From 8b1f0932bcc2d24e6b7d9be8023fd24bf70d5a3e Mon Sep 17 00:00:00 2001 From: alichinese Date: Tue, 25 Oct 2022 17:04:16 +0800 Subject: [PATCH 1/2] oebuilder: a convenient tool for openeuler building * you can run oebuilder [command] [option] to do what you want to do * command: * build: this command is to build image, you must do it with necessary param, eg: oebuilder build * -- arch_type: this is for cpu_platform for example aarch64-std,arm-std or aarch64-pro and so on * -- image_type: this is for build image type for example openeuler-image(standard) or openeuler-image-tiny(tiny) * -- workdir: this is for workspace, the workspace is to store source code and build directory, you can find what you want in this directory * then if you want build a standard image with platform aarch64-std in /home/ubuntu/Desktop/work workspace, you can run following command: oebuilder build aarch64-std openeuler-image /home/ubuntu/Desktop/work * note: You have to make sure that you have permissions in workspace Signed-off-by: lixinyu --- scripts/compile.sh | 4 +- scripts/init.sh | 23 ++++++ scripts/oebuilder/bin/oebuilder | 107 +++++++++++++++++++++++++ scripts/oebuilder/bin/oebuilder_docker | 51 ++++++++++++ scripts/oebuilder/bin/oebuilder_test | 10 +++ scripts/oebuilder/lib/docker_proxy.py | 76 ++++++++++++++++++ scripts/oebuilder/test/oebuilder.py | 42 ++++++++++ scripts/oebuilder/test/oebuilder.sh | 73 +++++++++++++++++ 8 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 scripts/init.sh create mode 100755 scripts/oebuilder/bin/oebuilder create mode 100755 scripts/oebuilder/bin/oebuilder_docker create mode 100755 scripts/oebuilder/bin/oebuilder_test create mode 100644 scripts/oebuilder/lib/docker_proxy.py create mode 100644 scripts/oebuilder/test/oebuilder.py create mode 100755 scripts/oebuilder/test/oebuilder.sh diff --git a/scripts/compile.sh b/scripts/compile.sh index 612872d5f6a..0bc1a4cad13 100644 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -150,6 +150,8 @@ download_pre_repo() fi SRC_DIR="$(realpath ${SRC_DIR})" + echo ${SRC_DIR} + POKY_DIR=${SRC_DIR}/yocto-poky test -d ${POKY_DIR} || git clone https://gitee.com/openeuler/yocto-poky.git -v ${POKY_DIR} -b ${SRC_BRANCH} --depth 1 } @@ -160,8 +162,8 @@ main() SRC_BRANCH="openEuler-22.09" GIT_PRE="https://gitee.com" GIT_SPACE="src-openeuler" - download_pre_repo get_build_info "$@" || return 1 + download_pre_repo set_env echo -e "Tip: You can now run 'bitbake ${BITBAKE_OPT}'.\n" } diff --git a/scripts/init.sh b/scripts/init.sh new file mode 100644 index 00000000000..12201d73f66 --- /dev/null +++ b/scripts/init.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +baseDir=`pwd` +oebuilderPath=$baseDir/oebuilder/bin +export PATH=$PATH:${oebuilderPath} +hello() +{ + cat < + -- arch_type: this is for cpu_platform for example aarch64-std,arm-std or aarch64-pro and so on + -- image_type: this is for build image type for example openeuler-image(standard) or openeuler-image-tiny(tiny) + -- workdir: this is for workspace, the workspace is to store source code and build directory, you can find what you want in this directory + then if you want build a standard image with platform aarch64-std in /home/ubuntu/Desktop/work workspace, you can run following command: + oebuilder build aarch64-std openeuler-image /home/ubuntu/Desktop/work + note: + You have to make sure that you have permissions in workspace +EOF +} +hello \ No newline at end of file diff --git a/scripts/oebuilder/bin/oebuilder b/scripts/oebuilder/bin/oebuilder new file mode 100755 index 00000000000..49a5c189f56 --- /dev/null +++ b/scripts/oebuilder/bin/oebuilder @@ -0,0 +1,107 @@ +#!/bin/bash + +check_build() +{ + if [ -z "${BUILD_ARCH}" ];then + return 1; + fi + + if [ -z "${BUILD_IMAGE}" ];then + return 1; + fi + + if [ -z "${BUILD_WORK}" ];then + return 1; + fi + + return 0 +} + +build() +{ + test -d ${BUILD_WORK} || mkdir -p ${BUILD_WORK} + + local srcDir=${BUILD_WORK}/src + test -d ${srcDir} || mkdir -p ${srcDir} + + local yoctoDir=${srcDir}/yocto-meta-openeuler + test -d ${yoctoDir} && rm -rf ${yoctoDir} + + local baseDir=`pwd`/../../yocto-meta-openeuler + cp -r ${baseDir} $srcDir + + local shellDir=${BUILD_WORK}/build.sh + test -f ${shellDir} && rm -f ${shellDir} + + local buildDir=build_${BUILD_ARCH}_${BUILD_IMAGE} + + cat > ${shellDir} << EOF +. /opt/buildtools/nativesdk/environment-setup-x86_64-pokysdk-linux +. /home/openeuler/work/src/yocto-meta-openeuler/scripts/compile.sh ${BUILD_ARCH} /home/openeuler/work/${buildDir} +bitbake ${BUILD_IMAGE} +EOF + + local volumes=${BUILD_WORK}:/home/openeuler/work + local image=swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container:latest + local command=/home/openeuler/work/build.sh + local user=openeuler + + oebuilder_docker exec -v ${volumes} -i ${image} -c ${command} -u ${user} +} + +use_agent() +{ + cat << EOF +Usage: +this is help information +EOF + + return 1 +} + +check_env() +{ + SYSTEM="" + + # check system type for installing next + grep -q "Ubuntu" /etc/os-release && SYSTEM="ubuntu" + grep -q "Centos" /etc/os-release && SYSTEM="centos" + + if [ `which docker` == "" ];then + if [ ${SYSTEM} == "ubuntu" ];then + sudo apt install docker + fi + if [ ${SYSTEM} == "centos" ];then + sudo yum install docker + fi + fi + + if [ `which python3` == "" ];then + if [ ${SYSTEM} == "ubuntu" ];then + sudo apt install python3 + fi + if [ ${SYSTEM} == "centos" ];then + sudo yum install python3 + fi + fi +} + +testing() +{ + oebuilder_test +} + +main() +{ + check_env + + if [ ${1} == "build" ];then + BUILD_ARCH="$2" + BUILD_IMAGE="$3" + BUILD_WORK="$4" + check_build || use_agent || return 1 + build + fi +} + +main "$@" \ No newline at end of file diff --git a/scripts/oebuilder/bin/oebuilder_docker b/scripts/oebuilder/bin/oebuilder_docker new file mode 100755 index 00000000000..887aa8bfa95 --- /dev/null +++ b/scripts/oebuilder/bin/oebuilder_docker @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse + +sys.path.insert(0, os.path.join(os.getcwd(), 'oebuilder')) +from lib.docker_proxy import DockerProxy, DockerImage + +def init_args(): + """ + init args + :return: + """ + parser = argparse.ArgumentParser() + parser.add_argument("action") + parser.add_argument("-v", type=str, dest="volumes", default="", required=False) + parser.add_argument("-c", type=str, dest="command", default="", required=False) + parser.add_argument("-i", type=str, dest="image", default="", required=False) + parser.add_argument("-u", type=str, dest="user", default="", required=False) + + return parser.parse_args() + + +def main(): + args = init_args() + dockerProxy = DockerProxy() + + action = args.action + if action == "exec": + volumes = args.volumes + if volumes != "": + volumes_split = volumes.split(",") + volumes = [] + for item in volumes_split: + volumes.append(item) + + imageName = args.image + command = args.command + user = args.user + + dockerProxy.run_container( + image_name = imageName, + commands = command, + volumes = volumes, + stream = True, + user = user + ) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/oebuilder/bin/oebuilder_test b/scripts/oebuilder/bin/oebuilder_test new file mode 100755 index 00000000000..b579133c995 --- /dev/null +++ b/scripts/oebuilder/bin/oebuilder_test @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +import os +import sys + +sys.path.insert(0, os.path.join(os.getcwd(), 'oebuilder')) + +from lib.docker_proxy import DockerProxy + +tt = DockerProxy() \ No newline at end of file diff --git a/scripts/oebuilder/lib/docker_proxy.py b/scripts/oebuilder/lib/docker_proxy.py new file mode 100644 index 00000000000..2aea821b88e --- /dev/null +++ b/scripts/oebuilder/lib/docker_proxy.py @@ -0,0 +1,76 @@ +import docker + +class DockerImage: + + REPOSITORY : str + TAG : str + IMAGE_ID : str + CREATED : str + SIZE : str + + +class DockerProxy(object): + + def __init__(self) -> None: + self._client = docker.from_env() + + def __del__(self) -> None: + self._client = None + + # 获取镜像列表 + def get_images(self) -> list[DockerImage]: + images = self._client.images + imageList = [] + for image in images.list(): + for tag in image.attrs['RepoTags']: + objImage = DockerImage + objImage.REPOSITORY = tag.split(':')[0] + objImage.TAG = tag.split(':')[1] + objImage.IMAGE_ID = image.short_id + objImage.CREATED = image.attrs["Created"] + objImage.SIZE = image.attrs['Size'] + imageList.append(objImage) + return imageList + + # 删除容器 + def rm_container(self, container_id : str = "", container = None): + self._client.api.remove_container(container_id) + + # 删除镜像 + def rm_image(self): + pass + + # 获取容器状态 + def get_container_info(self, container_id : str = "", container = None): + pass + + # 开始容器 + def start_container(self, container_id : str = "", container = None): + # containerObj = self.get_container_by_id(container_id = container_id) + self._client.api.start(container_id) + + # 获取某个正在运行容器 + def get_container_by_id(self, container_id): + return self._client.containers.get(container_id = container_id) + + # 创建容器（可执行命令，可执行脚本） + def run_container(self, image_name : str, commands = None, volumes = None, stream = False, work_dir = None, user = None): + container = self._client.containers.run( + image_name, + command = commands, + volumes = volumes, + stream = stream, + working_dir = work_dir, + user = user, + entrypoint = "bash", + detach = True, + remove = True, + stderr = True) + # print(container) + if stream == True: + for line in container.logs(stream = True): + print(line.decode()) + else: + print(container.logs().decode()) + + diff --git a/scripts/oebuilder/test/oebuilder.py b/scripts/oebuilder/test/oebuilder.py new file mode 100644 index 00000000000..647b9acb3e8 --- /dev/null +++ b/scripts/oebuilder/test/oebuilder.py @@ -0,0 +1,42 @@ +#!/bin/python3 +import os +import sys +import shutil +import subprocess + +workSpace = "/home/lisir/Desktop/project/openeuler/workspace" +if not os.path.exists(workSpace): + os.makedirs(workSpace) + +srcDir = os.path.join(workSpace, "src") +if not os.path.exists(srcDir): + os.mkdir(srcDir) + +yoctoDir = os.path.join(srcDir, "yocto-meta-openeuler") +if os.path.exists(yoctoDir): + shutil.rmtree(yoctoDir) + +# copy yocto-meta-openeuler to src directory +baseDir = os.path.abspath(os.path.join(sys.argv[0],"..","..","..")) +shutil.copytree(baseDir, yoctoDir) + +buildStr = ''' +. /opt/buildtools/nativesdk/environment-setup-x86_64-pokysdk-linux +. /home/jenkins/work/src/yocto-meta-openeuler/scripts/compile.sh aarch64-std /home/jenkins/work/build +bitbake openeuler-image +''' +buildDir = os.path.join(workSpace, "build.sh") +if os.path.exists(buildDir): + os.remove(buildDir) + +with open(buildDir, 'w') as f: + f.write(buildStr) + +imageName = "swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-ci-test:22.09" + +command = ''' +python oebuilder_docke.py exec -v {} -c {} -i {} -u {} +'''.format(workSpace+":/home/jenkins/work", "/home/jenkins/work/build.sh", imageName, "jenkins") +nowtime = subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) +for line in nowtime.stdout.truncate(): + print(line) \ No newline at end of file diff --git a/scripts/oebuilder/test/oebuilder.sh b/scripts/oebuilder/test/oebuilder.sh new file mode 100755 index 00000000000..9bcdbcd3c1f --- /dev/null +++ b/scripts/oebuilder/test/oebuilder.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +check_build() +{ + if [ -z "${BUILD_ARCH}" ];then + return 1; + fi + + if [ -z "${BUILD_IMAGE}" ];then + return 1; + fi + + if [ -z "${BUILD_WORK}" ];then + return 1; + fi + + return 0 +} + +build() +{ + test -d ${BUILD_WORK} || mkdir -p ${BUILD_WORK} + + local srcDir=${BUILD_WORK}/src + test -d ${srcDir} || mkdir -p ${srcDir} + + local yoctoDir=${srcDir}/yocto-meta-openeuler + test -d ${yoctoDir} && rm -rf ${yoctoDir} + + local baseDir=`cd $(cd $(dirname $0);pwd)/../..;pwd` + cp -r ${baseDir} $srcDir + + local shellDir=${BUILD_WORK}/build.sh + test -f ${shellDir} && rm -f ${shellDir} + + local buildDir=build_${BUILD_ARCH}_${BUILD_IMAGE} + + cat > ${shellDir} << EOF +. /opt/buildtools/nativesdk/environment-setup-x86_64-pokysdk-linux +. /home/jenkins/work/src/yocto-meta-openeuler/scripts/compile.sh ${BUILD_ARCH} /home/jenkins/work/${buildDir} +bitbake ${BUILD_IMAGE} +EOF + + local volumes=${BUILD_WORK}:/home/jenkins/work + local image=swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-ci-test:22.09 + local command=/home/jenkins/work/build.sh + local user=jenkins + + python oebuilder_docke.py exec -v ${volumes} -i ${image} -c ${command} -u ${user} +} + +use_agent() +{ + cat << EOF +Usage: +this is help information +EOF + + return 1 +} + +main() +{ + if [ ${1} == "build" ];then + BUILD_ARCH="$2" + BUILD_IMAGE="$3" + BUILD_WORK="$4" + check_build || use_agent || return 1 + build + fi +} + +main "$@" \ No newline at end of file -- Gitee From bde61f293a8e5a06a4d9ffc262f2c01c9b42d72e Mon Sep 17 00:00:00 2001 From: alichinese Date: Tue, 25 Oct 2022 17:04:16 +0800 Subject: [PATCH 2/2] oebuilder: add bitbake command * bitbake: this command is that you want to run bitbake and then has a peusdo-tty to nteract with mathine, when you run bitbake, you will open a build environment,it is a linux system standard but is to run some commands about yocto-poky, for example bitbake busybox -c do_fetch and so on Signed-off-by: lixinyu --- .gitignore | 2 +- scripts/init.sh | 9 +- scripts/oebuilder/bin/oebuilder | 146 ++++++++++++++++++++++--- scripts/oebuilder/bin/oebuilder_docker | 24 ++++ scripts/oebuilder/bin/oebuilder_test | 10 +- scripts/oebuilder/lib/docker_proxy.py | 41 +++++++ 6 files changed, 214 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index aa16c209e74..b5ad523e45d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,4 @@ docs/build *.swo # no vscode configuration -*.vscode +*.vscode \ No newline at end of file diff --git a/scripts/init.sh b/scripts/init.sh index 12201d73f66..751381d4916 100644 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -12,12 +12,19 @@ command: build: this command is to build image, you must do it with necessary param, eg: oebuilder build -- arch_type: this is for cpu_platform for example aarch64-std,arm-std or aarch64-pro and so on - -- image_type: this is for build image type for example openeuler-image(standard) or openeuler-image-tiny(tiny) + -- package: this is for build package for example openeuler-image(standard) or openeuler-image-tiny(tiny) and busybox and so on -- workdir: this is for workspace, the workspace is to store source code and build directory, you can find what you want in this directory then if you want build a standard image with platform aarch64-std in /home/ubuntu/Desktop/work workspace, you can run following command: oebuilder build aarch64-std openeuler-image /home/ubuntu/Desktop/work note: You have to make sure that you have permissions in workspace + setenv: this command is to set environment if you want use 'bitbake' command next + now environment param has following list: + -- WORK_SPACE: this is for workspace that you run bitbake and the source code and build result output + -- ARCH_TYPE: this is for cpu_platform for example aarch64-std,arm-std or aarch64-pro and so on + showenv: this command is showing environment that you set + bitbake: this command is that you want to run bitbake and then has a peusdo-tty to nteract with mathine, when you run bitbake, you will open a build environment, + it is a linux system standard but is to run some commands about yocto-poky, for example bitbake busybox -c do_fetch and so on EOF } hello \ No newline at end of file diff --git a/scripts/oebuilder/bin/oebuilder b/scripts/oebuilder/bin/oebuilder index 49a5c189f56..388498173df 100755 --- a/scripts/oebuilder/bin/oebuilder +++ b/scripts/oebuilder/bin/oebuilder @@ -1,5 +1,24 @@ #!/bin/bash +use_agent() +{ + + cat << EOF +Usage: +oebuilder command