From 1af87d177718ac51c31db4468873f7007c34c6e6 Mon Sep 17 00:00:00 2001 From: lixinyu Date: Thu, 11 Jan 2024 19:54:54 +0800 Subject: [PATCH 1/2] docker: optimize the invoke docker * all of us know that when we make a build task, of course most of time we base on docker container, we want to set some extra param at run docker, but oebuild start a docker with build-in params, thus it let us be in helpless, so this commit make a big alter teword to that matter. Also the code cover some other where, we optimize the const and alterd some place the const be at * in addition, we optimize the runqemu and deploy, the two function run base on docker, but when finished, we will wait a lot of time to destroy it, so we get a pollution and do it Signed-off-by: lixinyu --- src/oebuild/app/plugins/bitbake/base_build.py | 7 +- src/oebuild/app/plugins/bitbake/bitbake.py | 7 +- .../app/plugins/bitbake/in_container.py | 100 +++++++----------- src/oebuild/app/plugins/bitbake/in_host.py | 8 +- src/oebuild/app/plugins/deploy/com_target.py | 88 ++++++++------- src/oebuild/app/plugins/generate/generate.py | 30 +++--- src/oebuild/app/plugins/init/init.py | 11 +- src/oebuild/app/plugins/run_qemu/run_qemu.py | 36 +++---- src/oebuild/app/plugins/update/update.py | 6 +- src/oebuild/configure.py | 27 +++-- .../{app/plugins/bitbake => }/const.py | 35 ++++++ src/oebuild/docker_proxy.py | 82 ++++---------- src/oebuild/local_conf.py | 43 +++----- src/oebuild/parse_compile.py | 50 +++++++-- src/oebuild/parse_template.py | 48 ++++++--- src/oebuild/util.py | 31 ++---- 16 files changed, 310 insertions(+), 299 deletions(-) rename src/oebuild/{app/plugins/bitbake => }/const.py (38%) diff --git a/src/oebuild/app/plugins/bitbake/base_build.py b/src/oebuild/app/plugins/bitbake/base_build.py index 6e79fed..ccfec74 100644 --- a/src/oebuild/app/plugins/bitbake/base_build.py +++ b/src/oebuild/app/plugins/bitbake/base_build.py @@ -14,6 +14,7 @@ import os import oebuild.util as oebuild_util from oebuild.local_conf import LocalConf from oebuild.bblayers import BBLayers +import oebuild.const as oebuild_const class BaseBuild: ''' @@ -53,7 +54,7 @@ class BaseBuild: new_content = '' for line in old_content.split('\n'): line: str = line - if line.endswith(oebuild_util.BASH_END_FLAG) or line.replace(" ", '') == '': + if line.endswith(oebuild_const.BASH_END_FLAG) or line.replace(" ", '') == '': continue new_content = new_content + line + '\n' return new_content @@ -61,7 +62,7 @@ class BaseBuild: def _add_bashrc(self, content: str, line: str): if not content.endswith('\n'): content = content + '\n' - content = content + line + oebuild_util.BASH_END_FLAG + '\n' + content = content + line + oebuild_const.BASH_END_FLAG + '\n' return content @@ -69,6 +70,6 @@ class BaseBuild: new_content = self._restore_bashrc_content(old_content=old_content) for command in init_command: - new_content = new_content + command + oebuild_util.BASH_END_FLAG + '\n' + new_content = new_content + command + oebuild_const.BASH_END_FLAG + '\n' return new_content diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index 4c66bc5..4179271 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -14,6 +14,8 @@ import os import argparse import textwrap +from docker.errors import DockerException + from oebuild.command import OebuildCommand from oebuild.configure import Configure from oebuild.parse_compile import ParseCompile, CheckCompileError @@ -21,9 +23,8 @@ from oebuild.parse_env import ParseEnv import oebuild.util as oebuild_util from oebuild.app.plugins.bitbake.in_container import InContainer from oebuild.app.plugins.bitbake.in_host import InHost -from oebuild.parse_template import BUILD_IN_HOST from oebuild.m_log import logger -from docker.errors import DockerException +import oebuild.const as oebuild_const class Bitbake(OebuildCommand): ''' @@ -94,7 +95,7 @@ class Bitbake(OebuildCommand): parse_compile.check_with_version(self.configure.source_dir(), manifest_path=manifest_path) parse_env = ParseEnv(env_dir='.env') - if parse_compile.build_in == BUILD_IN_HOST: + if parse_compile.build_in == oebuild_const.BUILD_IN_HOST: in_host = InHost(self.configure) in_host.exec(parse_compile=parse_compile, command=command) return diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index 65c452c..dd88ade 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -15,15 +15,14 @@ import sys from docker.models.containers import Container,ExecResult -from oebuild.local_conf import NATIVE_GCC_DIR, SSTATE_CACHE from oebuild.parse_env import ParseEnv, EnvContainer from oebuild.docker_proxy import DockerProxy from oebuild.configure import Configure -from oebuild.parse_compile import ParseCompile +from oebuild.parse_compile import ParseCompile, DockerParam from oebuild.m_log import logger -import oebuild.app.plugins.bitbake.const as bitbake_const from oebuild.app.plugins.bitbake.base_build import BaseBuild import oebuild.util as oebuild_util +import oebuild.const as oebuild_const class InContainer(BaseBuild): ''' @@ -51,24 +50,16 @@ class InContainer(BaseBuild): logger.info("Bitbake starting ...") # check docker image if exists docker_proxy = DockerProxy() - if not docker_proxy.is_image_exists(parse_compile.docker_image): + if not docker_proxy.is_image_exists(parse_compile.docker_param.image): logger.error('''The docker image does not exists, please run fellow command: `oebuild update docker`''') return - self.deal_env_container( - env=parse_env, - toolchain_dir=parse_compile.toolchain_dir, - sstate_cache=parse_compile.sstate_cache, - docker_image=parse_compile.docker_image) + self.deal_env_container(env=parse_env,docker_param=parse_compile.docker_param) self.exec_compile(parse_compile=parse_compile, command=command) - def deal_env_container(self, - env: ParseEnv, - toolchain_dir=None, - sstate_cache = None, - docker_image = ""): + def deal_env_container(self, env: ParseEnv, docker_param: DockerParam): ''' This operation realizes the processing of the container, controls how the container is processed by parsing the env @@ -77,26 +68,15 @@ class InContainer(BaseBuild): are inconsistent, you need to create a new container, otherwise directly enable the sleeping container ''' - cwd_name = os.path.basename(os.getcwd()) - volumns = [] - volumns.append("/dev/net/tun:/dev/net/tun") - volumns.append(self.configure.source_dir() + ':' + oebuild_util.CONTAINER_SRC) - volumns.append(os.path.join(self.configure.build_dir(), cwd_name) - + ':' + - os.path.join(oebuild_util.CONTAINER_BUILD, cwd_name)) - if toolchain_dir is not None: - volumns.append(toolchain_dir + ":" + NATIVE_GCC_DIR) - - if sstate_cache is not None: - volumns.append(sstate_cache + ":" + SSTATE_CACHE) - if env.container is None \ or env.container.short_id is None \ or not self.client.is_container_exists(env.container.short_id): # judge which container - container:Container = self.client.container_run_simple( - image=docker_image, - volumes=volumns) # type: ignore + container:Container = self.client.create_container( + image = docker_param.image, + parameters=" ".join(docker_param.parameters), + volumes=docker_param.volumns, + command=docker_param.command) env_container = EnvContainer(container.short_id) env.set_env_container(env_container) @@ -126,7 +106,7 @@ class InContainer(BaseBuild): bblayers_dir = os.path.join(os.getcwd(), "conf", "bblayers.conf") self.add_bblayers( bblayers_dir=bblayers_dir, - pre_dir=oebuild_util.CONTAINER_SRC, + pre_dir=oebuild_const.CONTAINER_SRC, base_dir=self.configure.source_dir(), layers=parse_compile.layers) @@ -143,8 +123,8 @@ class InContainer(BaseBuild): res:ExecResult = self.client.container_exec_command( container=container, command="bash .bashrc", - user=oebuild_util.CONTAINER_USER, - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + user=oebuild_const.CONTAINER_USER, + work_space=f"/home/{oebuild_const.CONTAINER_USER}", demux=True) exit_code = 0 for line in res.output: @@ -156,12 +136,12 @@ class InContainer(BaseBuild): sys.exit(exit_code) else: content = self._get_bashrc_content(container=container) - for b_s in bitbake_const.BASH_BANNER.split('\n'): - b_s = f"echo {b_s}{oebuild_util.BASH_END_FLAG}" + for b_s in oebuild_const.BASH_BANNER.split('\n'): + b_s = f"echo {b_s}{oebuild_const.BASH_END_FLAG}" content = self._add_bashrc(content=content, line=b_s) self.update_bashrc(container=container, content=content) os.system( - f"docker exec -it -u {oebuild_util.CONTAINER_USER} {container.short_id} bash") + f"docker exec -it -u {oebuild_const.CONTAINER_USER} {container.short_id} bash") self.restore_bashrc(container=container) @@ -176,9 +156,9 @@ class InContainer(BaseBuild): res = self.client.container_exec_command( container=container, - command=f"bash /home/{oebuild_util.CONTAINER_USER}/.bashrc", - user=oebuild_util.CONTAINER_USER, - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + command=f"bash /home/{oebuild_const.CONTAINER_USER}/.bashrc", + user=oebuild_const.CONTAINER_USER, + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=False) if res.exit_code != 0: raise ValueError(res.output.decode()) @@ -188,8 +168,8 @@ class InContainer(BaseBuild): res = self.client.container_exec_command( container=container, user='root', - command=f"id {oebuild_util.CONTAINER_USER}", - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + command=f"id {oebuild_const.CONTAINER_USER}", + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=False) if res.exit_code != 0: raise ValueError("check docker user id faild") @@ -198,19 +178,19 @@ class InContainer(BaseBuild): cuids = res_cont.split(' ') # get uid from container in default user - pattern = re.compile(r'(?<=uid=)\d{1,}(?=\(' + oebuild_util.CONTAINER_USER + r'\))') + pattern = re.compile(r'(?<=uid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))') match_uid = pattern.search(cuids[0]) if match_uid: cuid = match_uid.group() else: - raise ValueError(f"can not get container {oebuild_util.CONTAINER_USER} uid") + raise ValueError(f"can not get container {oebuild_const.CONTAINER_USER} uid") # get gid from container in default user - pattern = re.compile(r'(?<=gid=)\d{1,}(?=\(' + oebuild_util.CONTAINER_USER + r'\))') + pattern = re.compile(r'(?<=gid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))') match_gid = pattern.search(cuids[1]) if match_gid: cgid = match_gid.group() else: - raise ValueError(f"can not get container {oebuild_util.CONTAINER_USER} gid") + raise ValueError(f"can not get container {oebuild_const.CONTAINER_USER} gid") # judge host uid and gid are same with container uid and gid # if not same and change container uid and gid equal to host's uid and gid @@ -223,16 +203,16 @@ class InContainer(BaseBuild): self.client.container_exec_command( container=container, user='root', - command=f"usermod -u {uid} {oebuild_util.CONTAINER_USER}", - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + command=f"usermod -u {uid} {oebuild_const.CONTAINER_USER}", + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=False) def _change_container_gid(self, container: Container, gid: int): self.client.container_exec_command( container=container, user='root', - command=f"groupmod -g {gid} {oebuild_util.CONTAINER_USER}", - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + command=f"groupmod -g {gid} {oebuild_const.CONTAINER_USER}", + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=False) def _install_sudo(self, container: Container): @@ -242,7 +222,7 @@ class InContainer(BaseBuild): container=container, user='root', command="which sudo", - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=False ) if resp.exit_code != 0: @@ -255,7 +235,7 @@ class InContainer(BaseBuild): container=container, user='root', command=r"sed -i 's/repo.openeuler.org/mirrors.huaweicloud.com\/openeuler/g' /etc/yum.repos.d/openEuler.repo", - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=False ) @@ -264,7 +244,7 @@ class InContainer(BaseBuild): container=container, user='root', command=f"yum install {software} -y", - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=True ) for line in resp.output: @@ -281,16 +261,16 @@ class InContainer(BaseBuild): # get host proxy information and set in container init_proxy_command = "" - host_proxy = oebuild_util.get_host_proxy(oebuild_util.PROXY_LIST) + host_proxy = oebuild_util.get_host_proxy(oebuild_const.PROXY_LIST) for key, value in host_proxy.items(): key_git = key.replace('_', '.') command = f'export {key}={value}; git config --global {key_git} {value};' init_proxy_command = f'{init_proxy_command} {command}' - init_sdk_command = f'. {oebuild_util.NATIVESDK_DIR}/{oebuild_util.get_nativesdk_environment(container=container)}' - set_template = f'export TEMPLATECONF="{oebuild_util.CONTAINER_SRC}/yocto-meta-openeuler/.oebuild"' - init_oe_comand = f'. {oebuild_util.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ - {oebuild_util.CONTAINER_BUILD}/{build_dir_name}' + init_sdk_command = f'. {oebuild_const.NATIVESDK_DIR}/{oebuild_util.get_nativesdk_environment(container=container)}' + set_template = f'export TEMPLATECONF="{oebuild_const.CONTAINER_SRC}/yocto-meta-openeuler/.oebuild"' + init_oe_comand = f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ + {oebuild_const.CONTAINER_BUILD}/{build_dir_name}' init_command = [init_proxy_command, init_sdk_command, set_template, init_oe_comand] new_content = self._init_bashrc_content(content, init_command) @@ -306,9 +286,9 @@ class InContainer(BaseBuild): self.client.copy_to_container( container=container, source_path=tmp_file, - to_path=f'/home/{oebuild_util.CONTAINER_USER}') + to_path=f'/home/{oebuild_const.CONTAINER_USER}') container.exec_run( - cmd=f"mv /home/{oebuild_util.CONTAINER_USER}/{tmp_file} /home/{oebuild_util.CONTAINER_USER}/.bashrc", + cmd=f"mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINER_USER}/.bashrc", user="root" ) os.remove(tmp_file) @@ -325,7 +305,7 @@ class InContainer(BaseBuild): def _get_bashrc_content(self, container: Container): res = self.client.container_exec_command( container=container, - command=f"cat /home/{oebuild_util.CONTAINER_USER}/.bashrc", + command=f"cat /home/{oebuild_const.CONTAINER_USER}/.bashrc", user="root", work_space=None, stream=False) diff --git a/src/oebuild/app/plugins/bitbake/in_host.py b/src/oebuild/app/plugins/bitbake/in_host.py index 1f2db45..f9d1c4c 100644 --- a/src/oebuild/app/plugins/bitbake/in_host.py +++ b/src/oebuild/app/plugins/bitbake/in_host.py @@ -19,9 +19,9 @@ from oebuild.local_conf import NativesdkNotExist, NativesdkNotValid from oebuild.configure import Configure from oebuild.parse_compile import ParseCompile from oebuild.m_log import logger -import oebuild.app.plugins.bitbake.const as bitbake_const from oebuild.app.plugins.bitbake.base_build import BaseBuild import oebuild.util as oebuild_util +import oebuild.const as oebuild_const class InHost(BaseBuild): ''' @@ -93,11 +93,11 @@ initialization operations''') else: # run in Interactive mode banner_list = [] - for b_s in bitbake_const.BASH_BANNER.split('\n'): - b_s = f"echo {b_s}{oebuild_util.BASH_END_FLAG}" + for b_s in oebuild_const.BASH_BANNER.split('\n'): + b_s = f"echo {b_s}{oebuild_const.BASH_END_FLAG}" banner_list.append(b_s) self._append_build_sh(str_list=banner_list, build_dir= os.getcwd()) - append_str = f"sed -i '/{oebuild_util.BASH_END_FLAG}/d' $HOME/.bashrc" + append_str = f"sed -i '/{oebuild_const.BASH_END_FLAG}/d' $HOME/.bashrc" self._append_build_sh(str_list = [append_str], build_dir= os.getcwd()) build_sh_dir = os.path.join(os.getcwd(), 'build.sh') diff --git a/src/oebuild/app/plugins/deploy/com_target.py b/src/oebuild/app/plugins/deploy/com_target.py index 7902a86..d4f8db0 100644 --- a/src/oebuild/app/plugins/deploy/com_target.py +++ b/src/oebuild/app/plugins/deploy/com_target.py @@ -16,13 +16,14 @@ import re import logging from docker.models.containers import Container +from docker.errors import DockerException from oebuild.docker_proxy import DockerProxy from oebuild.configure import Configure import oebuild.util as oebuild_util -from oebuild.parse_compile import ParseCompile, CheckCompileError,BUILD_IN_DOCKER -from oebuild.parse_env import ParseEnv,EnvContainer +from oebuild.parse_compile import ParseCompile, CheckCompileError from oebuild.m_log import logger +import oebuild.const as oebuild_const logger = logging.getLogger() @@ -37,6 +38,19 @@ class ComTarget: self.work_dir = os.getcwd() self.old_bashrc = None + def __del__(self): + if self.client is not None: + try: + container = self.client.get_container(self.container_id) + self.client.delete_container(container=container, is_force=True) + except DockerException: + print(f""" +the container {self.container_id} failed to be destroyed, please run + +`docker rm {self.container_id}` + +""") + def exec(self, str_args: str, fun): self.client = DockerProxy() @@ -59,20 +73,19 @@ class ComTarget: sys.exit(-1) logger.info("Initializing environment, please wait ...") - parse_env = ParseEnv(env_dir='.env') - self.deal_env_container(parse_env, oebuild_util.DEFAULT_DOCKER) + self.deal_env_container(oebuild_const.DEFAULT_DOCKER) container:Container = self.client.get_container(self.container_id) self._make_and_copy_lib(container=container) self.bak_bash(container=container) self.init_bash(container=container) content = self._get_bashrc_content(container=container) content = oebuild_util.add_bashrc(content=content, line=f"export PATH=$PATH:/home/openeuler/{TARGET_DIR_NAME}") - content = oebuild_util.add_bashrc(content=content, line=f"mv -f /home/{oebuild_util.CONTAINER_USER}/{self.old_bashrc} /home/{oebuild_util.CONTAINER_USER}/.bashrc") + content = oebuild_util.add_bashrc(content=content, line=f"mv -f /home/{oebuild_const.CONTAINER_USER}/{self.old_bashrc} /home/{oebuild_const.CONTAINER_USER}/.bashrc") content = oebuild_util.add_bashrc(content=content, line=f"{TARGET_SCRIPT_NAME} {fun} {str_args}") print(f"{TARGET_SCRIPT_NAME} {str_args}") content = oebuild_util.add_bashrc(content=content, line=f"rm -rf /home/openeuler/{TARGET_DIR_NAME} && exit") self.update_bashrc(container=container, content=content) - os.system(f"docker exec -it -u {oebuild_util.CONTAINER_USER} {container.short_id} bash") + os.system(f"docker exec -it -u {oebuild_const.CONTAINER_USER} {container.short_id} bash") def _check_conf_directory(self,): # check if exists local.conf @@ -93,7 +106,7 @@ class ComTarget: except CheckCompileError as c_e: logger.error(str(c_e)) sys.exit(-1) - if parse_compile.build_in != BUILD_IN_DOCKER: + if parse_compile.build_in != oebuild_const.BUILD_IN_DOCKER: return False return True @@ -119,7 +132,7 @@ class ComTarget: self.client.copy_to_container(container=container, source_path=lib_path, to_path="/home/openeuler/") container.exec_run(f"chmod 755 /home/openeuler/{TARGET_DIR_NAME}/{TARGET_SCRIPT_NAME}") - def deal_env_container(self, env: ParseEnv,docker_image:str): + def deal_env_container(self,docker_image:str): ''' This operation realizes the processing of the container, controls how the container is processed by parsing the env @@ -131,24 +144,19 @@ class ComTarget: cwd_name = os.path.basename(self.work_dir) volumns = [] volumns.append("/dev/net/tun:/dev/net/tun") - volumns.append(self.configure.source_dir() + ':' + oebuild_util.CONTAINER_SRC) + volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) volumns.append(os.path.join(self.configure.build_dir(), cwd_name) + ':' + - os.path.join(oebuild_util.CONTAINER_BUILD, cwd_name)) - - if env.container is None \ - or env.container.short_id is None \ - or not self.client.is_container_exists(env.container.short_id): - # judge which container - container:Container = self.client.container_run_simple( - image=docker_image, - volumes=volumns) # type: ignore + os.path.join(oebuild_const.CONTAINER_BUILD, cwd_name)) - env_container = EnvContainer(container.short_id) - env.set_env_container(env_container) - env.export_env() + parameters = "-itd" + container:Container = self.client.create_container( + image=docker_image, + parameters=parameters, + volumes=volumns, + command="bash") - self.container_id = env.container.short_id + self.container_id = container.short_id container:Container = self.client.get_container(self.container_id) # type: ignore if not self.client.is_container_running(container): self.client.start_container(container) @@ -160,7 +168,7 @@ class ComTarget: old_bash = oebuild_util.generate_random_str(6) self.client.container_exec_command( container=container, - command=f"cp /home/{oebuild_util.CONTAINER_USER}/.bashrc /home/{oebuild_util.CONTAINER_USER}/{old_bash}", + command=f"cp /home/{oebuild_const.CONTAINER_USER}/.bashrc /home/{oebuild_const.CONTAINER_USER}/{old_bash}", user="root", work_space=None, stream=False) @@ -175,10 +183,10 @@ class ComTarget: self._check_change_ugid(container=container) # read container default user .bashrc content content = self._get_bashrc_content(container=container) - init_sdk_command = f'. {oebuild_util.NATIVESDK_DIR}/{oebuild_util.get_nativesdk_environment(container=container)}' + init_sdk_command = f'. {oebuild_const.NATIVESDK_DIR}/{oebuild_util.get_nativesdk_environment(container=container)}' build_dir_name = os.path.basename(self.work_dir) - init_oe_command = f'. {oebuild_util.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ - {oebuild_util.CONTAINER_BUILD}/{build_dir_name}' + init_oe_command = f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env \ + {oebuild_const.CONTAINER_BUILD}/{build_dir_name}' init_command = [init_sdk_command, init_oe_command] new_content = oebuild_util.init_bashrc_content(content, init_command) self.update_bashrc(container=container, content=new_content) @@ -186,7 +194,7 @@ class ComTarget: def _get_bashrc_content(self, container: Container): content = self.client.container_exec_command( container=container, - command=f"cat /home/{oebuild_util.CONTAINER_USER}/.bashrc", + command=f"cat /home/{oebuild_const.CONTAINER_USER}/.bashrc", user="root", work_space=None, stream=False).output @@ -203,9 +211,9 @@ class ComTarget: self.client.copy_to_container( container=container, source_path=tmp_file, - to_path=f'/home/{oebuild_util.CONTAINER_USER}') + to_path=f'/home/{oebuild_const.CONTAINER_USER}') container.exec_run( - cmd=f"mv /home/{oebuild_util.CONTAINER_USER}/{tmp_file} /home/{oebuild_util.CONTAINER_USER}/.bashrc", + cmd=f"mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINER_USER}/.bashrc", user="root" ) os.remove(tmp_file) @@ -233,7 +241,7 @@ class ComTarget: new_content = '' for line in old_content.split('\n'): line: str = line - if line.endswith(oebuild_util.BASH_END_FLAG) or line.replace(" ", '') == '': + if line.endswith(oebuild_const.BASH_END_FLAG) or line.replace(" ", '') == '': continue new_content = new_content + line + '\n' return new_content @@ -242,8 +250,8 @@ class ComTarget: res = self.client.container_exec_command( container=container, user='root', - command=f"id {oebuild_util.CONTAINER_USER}", - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + command=f"id {oebuild_const.CONTAINER_USER}", + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=False) if res.exit_code != 0: raise ValueError("check docker user id faild") @@ -252,19 +260,19 @@ class ComTarget: cuids = res_cont.split(' ') # get uid from container in default user - pattern = re.compile(r'(?<=uid=)\d{1,}(?=\(' + oebuild_util.CONTAINER_USER + r'\))') + pattern = re.compile(r'(?<=uid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))') match_uid = pattern.search(cuids[0]) if match_uid: cuid = match_uid.group() else: - raise ValueError(f"can not get container {oebuild_util.CONTAINER_USER} uid") + raise ValueError(f"can not get container {oebuild_const.CONTAINER_USER} uid") # get gid from container in default user - pattern = re.compile(r'(?<=gid=)\d{1,}(?=\(' + oebuild_util.CONTAINER_USER + r'\))') + pattern = re.compile(r'(?<=gid=)\d{1,}(?=\(' + oebuild_const.CONTAINER_USER + r'\))') match_gid = pattern.search(cuids[1]) if match_gid: cgid = match_gid.group() else: - raise ValueError(f"can not get container {oebuild_util.CONTAINER_USER} gid") + raise ValueError(f"can not get container {oebuild_const.CONTAINER_USER} gid") # judge host uid and gid are same with container uid and gid # if not same and change container uid and gid equal to host's uid and gid @@ -277,14 +285,14 @@ class ComTarget: self.client.container_exec_command( container=container, user='root', - command=f"usermod -u {uid} {oebuild_util.CONTAINER_USER}", - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + command=f"usermod -u {uid} {oebuild_const.CONTAINER_USER}", + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=False) def _change_container_gid(self, container: Container, gid: int): self.client.container_exec_command( container=container, user='root', - command=f"groupmod -g {gid} {oebuild_util.CONTAINER_USER}", - work_space=f"/home/{oebuild_util.CONTAINER_USER}", + command=f"groupmod -g {gid} {oebuild_const.CONTAINER_USER}", + work_space=f"/home/{oebuild_const.CONTAINER_USER}", stream=False) \ No newline at end of file diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index f24170b..96d3f85 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -23,9 +23,10 @@ from oebuild.command import OebuildCommand import oebuild.util as oebuild_util from oebuild.parse_compile import ParseCompile,CheckCompileError from oebuild.configure import Configure, YoctoEnv -from oebuild.parse_template import BaseParseTemplate, ParseTemplate, BUILD_IN_DOCKER, BUILD_IN_HOST +from oebuild.parse_template import BaseParseTemplate, ParseTemplate from oebuild.m_log import logger, INFO_COLOR from oebuild.check_docker_tag import CheckDockerTag +import oebuild.const as oebuild_const class Generate(OebuildCommand): ''' @@ -117,7 +118,7 @@ class Generate(OebuildCommand): this param is for external nativesdk dir, the param will be useful when you want to build in host ''' ) - + parser.add_argument('-tag', '--docker_tag', dest='docker_tag', default = '', help=''' when build in docker, the param can be point docker image @@ -137,11 +138,13 @@ class Generate(OebuildCommand): this param is set openeuler_fetch in local.conf, the default value is enable, if set -df, the OPENEULER_FETCH will set to 'disable' ''') - parser.add_argument('-b_in', '--build_in', dest='build_in', choices=[BUILD_IN_DOCKER, BUILD_IN_HOST], - default = BUILD_IN_DOCKER, help=''' + parser.add_argument('-b_in', + '--build_in', + dest='build_in', + choices=[oebuild_const.BUILD_IN_DOCKER, oebuild_const.BUILD_IN_HOST], + default = oebuild_const.BUILD_IN_DOCKER, help=''' This parameter marks the mode at build time, and is built in the container by docker - ''' - ) + ''') return parser @@ -181,15 +184,15 @@ class Generate(OebuildCommand): logger.error(str(e)) return - build_in = BUILD_IN_DOCKER - if args.build_in == BUILD_IN_HOST: + build_in = oebuild_const.BUILD_IN_DOCKER + if args.build_in == oebuild_const.BUILD_IN_HOST: try: self._check_param_in_host(args=args) except ValueError as v_e: logger.error(str(v_e)) return self.nativesdk_dir = args.nativesdk_dir - build_in = BUILD_IN_HOST + build_in = oebuild_const.BUILD_IN_HOST if args.toolchain_dir != '': self.toolchain_dir = args.toolchain_dir @@ -268,7 +271,8 @@ class Generate(OebuildCommand): docker_image = f"{oebuild_config.docker.repo_url}:{docker_tag}" out_dir = pathlib.Path(os.path.join(build_dir,'compile.yaml')) - oebuild_util.write_yaml(out_dir, parser_template.generate_template( + + oebuild_util.write_yaml(out_dir, parser_template.generate_compile_conf( nativesdk_dir= self.nativesdk_dir, toolchain_dir= self.toolchain_dir, build_in=build_in, @@ -276,7 +280,9 @@ class Generate(OebuildCommand): tmp_dir = self.tmp_dir, datetime = args.datetime, is_disable_fetch = args.is_disable_fetch, - docker_image=docker_image + docker_image=docker_image, + src_dir=self.configure.source_dir(), + compile_dir=build_dir )) self._print_generate(build_dir=build_dir) @@ -349,7 +355,7 @@ oebuild bitbake # detects if a build directory already exists if os.path.exists(build_dir): - logger.warn("the build directory %s already exists", build_dir) + logger.warning("the build directory %s already exists", build_dir) while True: in_res = input(""" do you want to overwrite it? the overwrite action will replace the compile.yaml diff --git a/src/oebuild/app/plugins/init/init.py b/src/oebuild/app/plugins/init/init.py index 5680b7c..e342058 100644 --- a/src/oebuild/app/plugins/init/init.py +++ b/src/oebuild/app/plugins/init/init.py @@ -18,8 +18,9 @@ import sys from oebuild.command import OebuildCommand import oebuild.util as oebuild_util -from oebuild.configure import Configure, YOCTO_META_OPENEULER, ConfigBasicRepo, CONFIG, COMPILE_YAML, Config +from oebuild.configure import Configure, ConfigBasicRepo, Config from oebuild.m_log import logger +import oebuild.const as oebuild_const class Init(OebuildCommand): ''' @@ -101,12 +102,12 @@ class Init(OebuildCommand): os.chdir(args.directory) oebuild_config:Config = self.configure.parse_oebuild_config() - yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[YOCTO_META_OPENEULER] + yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] if args.yocto_remote_url is not None: yocto_config.remote_url = args.yocto_remote_url if args.branch is not None: yocto_config.branch = args.branch - oebuild_config.basic_repo[YOCTO_META_OPENEULER] = yocto_config + oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] = yocto_config self.configure.update_oebuild_config(oebuild_config) @@ -170,7 +171,7 @@ please execute the follow commands next ''' try: config = oebuild_util.get_config_yaml_dir() - shutil.copyfile(config, os.path.join(updir, CONFIG)) + shutil.copyfile(config, os.path.join(updir, oebuild_const.CONFIG)) except FileNotFoundError: logger.error("mkdir config faild") @@ -181,6 +182,6 @@ please execute the follow commands next ''' try: compil = oebuild_util.get_compile_yaml_dir() - shutil.copyfile(compil, os.path.join(updir, COMPILE_YAML)) + shutil.copyfile(compil, os.path.join(updir, oebuild_const.COMPILE_YAML)) except FileNotFoundError: logger.error("mkdir compile.yaml.sample failed") diff --git a/src/oebuild/app/plugins/run_qemu/run_qemu.py b/src/oebuild/app/plugins/run_qemu/run_qemu.py index 141dee3..a51c8a0 100644 --- a/src/oebuild/app/plugins/run_qemu/run_qemu.py +++ b/src/oebuild/app/plugins/run_qemu/run_qemu.py @@ -23,6 +23,7 @@ from oebuild.docker_proxy import DockerProxy from oebuild.command import OebuildCommand from oebuild.configure import Configure from oebuild.m_log import logger +import oebuild.const as oebuild_const class RunQemu(OebuildCommand): ''' @@ -45,15 +46,9 @@ class RunQemu(OebuildCommand): def __del__(self): if self.client is not None: - print(""" - - container is being destroyed, please wait ... - -""") try: container = self.client.get_container(self.container_id) - self.client.stop_container(container=container) - self.client.delete_container(container=container) + self.client.delete_container(container=container, is_force=True) except DockerException: print(f""" the container {self.container_id} failed to be destroyed, please run @@ -85,9 +80,7 @@ the container {self.container_id} failed to be destroyed, please run docker_image = self.get_docker_image() self._check_qemu_ifup() self.deal_env_container(docker_image=docker_image) - # container:Container = self.client.get_container(self.container_id) - # for index, param in enumerate(unknown): if param.startswith("qemuparams"): unknown[index] = "qemuparams=\""+param.split("=")[1]+"\"" @@ -103,12 +96,12 @@ the container {self.container_id} failed to be destroyed, please run self.bak_bash(container=container) self.init_bash(container=container) content = self._get_bashrc_content(container=container) - qemu_helper_usr = oebuild_util.CONTAINER_BUILD+"/tmp/work/x86_64-linux/qemu-helper-native/1.0-r1/recipe-sysroot-native/usr" - qemu_helper_dir = oebuild_util.CONTAINER_BUILD+"/tmp/work/x86_64-linux/qemu-helper-native" + qemu_helper_usr = oebuild_const.CONTAINER_BUILD+"/tmp/work/x86_64-linux/qemu-helper-native/1.0-r1/recipe-sysroot-native/usr" + qemu_helper_dir = oebuild_const.CONTAINER_BUILD+"/tmp/work/x86_64-linux/qemu-helper-native" STAGING_BINDIR_NATIVE = f""" if [ ! -d {qemu_helper_usr} ];then mkdir -p {qemu_helper_usr} - chown -R {oebuild_util.CONTAINER_USER}:{oebuild_util.CONTAINER_USER} {qemu_helper_dir} + chown -R {oebuild_const.CONTAINER_USER}:{oebuild_const.CONTAINER_USER} {qemu_helper_dir} ln -s /opt/buildtools/nativesdk/sysroots/x86_64-pokysdk-linux/usr/bin {qemu_helper_usr} fi """ @@ -155,15 +148,18 @@ now, you can continue run `oebuild runqemu` in compile directory volumns = [] volumns.append("/dev/net/tun:/dev/net/tun") volumns.append("/etc/qemu-ifup:/etc/qemu-ifup") - volumns.append(self.work_dir + ':' + oebuild_util.CONTAINER_BUILD) - volumns.append(self.configure.source_dir() + ':' + oebuild_util.CONTAINER_SRC) - container:Container = self.client.container_run_simple( + volumns.append(self.work_dir + ':' + oebuild_const.CONTAINER_BUILD) + volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) + + parameters = "-itd --privileged" + container:Container = self.client.create_container( image=docker_image, + parameters=parameters, volumes=volumns, - is_priv=True) # type: ignore + command="bash") self.container_id = container.short_id - container:Container = self.client.get_container(self.container_id) # type: ignore + container:Container = self.client.get_container(self.container_id) if not self.client.is_container_running(container): self.client.start_container(container) @@ -171,7 +167,7 @@ now, you can continue run `oebuild runqemu` in compile directory ''' this is function is to get openeuler docker image automatic ''' - return oebuild_util.DEFAULT_DOCKER + return oebuild_const.DEFAULT_DOCKER def bak_bash(self, container: Container): ''' @@ -195,8 +191,8 @@ now, you can continue run `oebuild runqemu` in compile directory # read container default user .bashrc content content = self._get_bashrc_content(container=container) - init_sdk_command = f'. {oebuild_util.NATIVESDK_DIR}/{oebuild_util.get_nativesdk_environment(container=container)}' - init_oe_command = f'. {oebuild_util.CONTAINER_SRC}/yocto-poky/oe-init-build-env {oebuild_util.CONTAINER_BUILD}' + init_sdk_command = f'. {oebuild_const.NATIVESDK_DIR}/{oebuild_util.get_nativesdk_environment(container=container)}' + init_oe_command = f'. {oebuild_const.CONTAINER_SRC}/yocto-poky/oe-init-build-env {oebuild_const.CONTAINER_BUILD}' init_command = [init_sdk_command, init_oe_command] new_content = oebuild_util.init_bashrc_content(content, init_command) self.update_bashrc(container=container, content=new_content) diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py index 7c742b7..6dcc826 100644 --- a/src/oebuild/app/plugins/update/update.py +++ b/src/oebuild/app/plugins/update/update.py @@ -22,11 +22,11 @@ import oebuild.util as oebuild_util from oebuild.command import OebuildCommand from oebuild.parse_template import OebuildRepo from oebuild.parse_compile import ParseCompile -from oebuild.configure import Configure, ConfigBasicRepo, YOCTO_META_OPENEULER, YoctoEnv +from oebuild.configure import Configure, ConfigBasicRepo, YoctoEnv from oebuild.docker_proxy import DockerProxy from oebuild.ogit import OGit from oebuild.check_docker_tag import CheckDockerTag - +import oebuild.const as oebuild_const from oebuild.m_log import logger class Update(OebuildCommand): @@ -171,7 +171,7 @@ class Update(OebuildCommand): embedded/src/yocto-meta-openeuler not exists, so just clone from config setting. ''' oebuild_config = self.configure.parse_oebuild_config() - yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[YOCTO_META_OPENEULER] + yocto_config:ConfigBasicRepo = oebuild_config.basic_repo[oebuild_const.YOCTO_META_OPENEULER] local_dir = os.path.join(self.configure.source_dir(), yocto_config.path) yocto_repo = OGit(repo_dir=local_dir, diff --git a/src/oebuild/configure.py b/src/oebuild/configure.py index fffb2bd..9a71322 100644 --- a/src/oebuild/configure.py +++ b/src/oebuild/configure.py @@ -16,18 +16,13 @@ import pathlib from dataclasses import dataclass import oebuild.util as oebuild_util +import oebuild.const as oebuild_const PathType = Union[str, os.PathLike] -YOCTO_META_OPENEULER = "yocto_meta_openeuler" -YOCTO_POKY = "yocto-poky" -CONFIG = "config" -COMPILE_YAML = "compile.yaml.sample" - class OebuildNotFound(RuntimeError): '''Neither the current directory nor any parent has a oebuild workspace.''' - @dataclass class ConfigContainer: ''' @@ -128,7 +123,7 @@ class Configure: ''' config = Configure.parse_oebuild_config() basic_config = config.basic_repo - yocto_config:ConfigBasicRepo = basic_config[YOCTO_META_OPENEULER] + yocto_config:ConfigBasicRepo = basic_config[oebuild_const.YOCTO_META_OPENEULER] yocto_dir = yocto_config.path return os.path.join(Configure.source_dir(), yocto_dir) @@ -137,7 +132,7 @@ class Configure: ''' return src/yocto-poky path ''' - return os.path.join(Configure.source_dir(), YOCTO_POKY) + return os.path.join(Configure.source_dir(), oebuild_const.YOCTO_POKY) @staticmethod def yocto_bak_dir(): @@ -168,7 +163,8 @@ class Configure: the file path is {WORKSPACE}.oebuild/config ''' - config = oebuild_util.read_yaml(yaml_dir = pathlib.Path(Configure.oebuild_dir(), CONFIG)) + config = oebuild_util.read_yaml( + yaml_dir = pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG)) tag_map = {} for key, value in config['docker']['tag_map'].items(): @@ -209,14 +205,17 @@ class Configure: 'branch': repo.branch} try: - oebuild_util.write_yaml(yaml_dir = pathlib.Path(Configure.oebuild_dir(), CONFIG), - data=data) + oebuild_util.write_yaml( + yaml_dir = pathlib.Path(Configure.oebuild_dir(), oebuild_const.CONFIG), + data=data) return True except TypeError: return False class YoctoEnv: - + ''' + the YoctoEnv class is used for yocto-meta-openeuler/.oebuild/env.yaml + ''' @staticmethod def get_docker_image(yocto_dir): ''' @@ -228,9 +227,9 @@ class YoctoEnv: env_path = os.path.join(yocto_dir,".oebuild/env.yaml") if not os.path.exists(env_path): return None - + env_parse = oebuild_util.read_yaml(pathlib.Path(env_path)) if "docker_image" in env_parse: return str(env_parse['docker_image']) - + return None diff --git a/src/oebuild/app/plugins/bitbake/const.py b/src/oebuild/const.py similarity index 38% rename from src/oebuild/app/plugins/bitbake/const.py rename to src/oebuild/const.py index 18f28e8..195342d 100644 --- a/src/oebuild/app/plugins/bitbake/const.py +++ b/src/oebuild/const.py @@ -10,8 +10,43 @@ MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. See the Mulan PSL v2 for more details. ''' +# used for util.py +CONFIG_YAML = 'config.yaml' +UPGRADE_YAML = 'upgrade.yaml' +COMPILE_YAML = 'compile.yaml.sample' +BASH_END_FLAG = " ###!!!###" +CONTAINER_USER = "openeuler" +CONTAINER_BUILD = '/home/openeuler/build' +DEFAULT_DOCKER = "swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container:latest" +CONTAINER_SRC = '/usr1/openeuler/src' +CONTAINER_USER = "openeuler" +NATIVESDK_DIR = "/opt/buildtools/nativesdk" +PROXY_LIST = ['http_proxy', 'https_proxy'] + +# used for local_conf +NATIVESDK_DIR_NAME = "OPENEULER_NATIVESDK_SYSROOT" +OPENEULER_SP_DIR = "OPENEULER_SP_DIR" +SSTATE_MIRRORS = "SSTATE_MIRRORS" +SSTATE_DIR = "SSTATE_DIR" +TMP_DIR = "TMPDIR" + +NATIVE_GCC_DIR = '/usr1/openeuler/native_gcc' +SSTATE_CACHE = '/usr1/openeuler/sstate-cache' + +# used for bitbake/in_container.py BASH_BANNER = ''' Welcome to the openEuler Embedded build environment, where you can run 'bitbake openeuler-image' to build standard images ''' + +# used for configure.py +YOCTO_META_OPENEULER = "yocto_meta_openeuler" +YOCTO_POKY = "yocto-poky" +CONFIG = "config" +COMPILE_YAML = "compile.yaml.sample" + +# used for parse_templete.py +PLATFORM = 'platform' +BUILD_IN_DOCKER = "docker" +BUILD_IN_HOST = "host" diff --git a/src/oebuild/docker_proxy.py b/src/oebuild/docker_proxy.py index d6e75f0..9ceba04 100644 --- a/src/oebuild/docker_proxy.py +++ b/src/oebuild/docker_proxy.py @@ -13,11 +13,15 @@ See the Mulan PSL v2 for more details. import os from io import BytesIO import tarfile +import subprocess +import sys import docker from docker.errors import ImageNotFound, NotFound from docker.models.containers import Container +from oebuild.m_log import logger + class DockerProxy: ''' a object just be wrapper again to run docker command easily @@ -108,13 +112,13 @@ class DockerProxy: container.stop() @staticmethod - def delete_container(container: Container): + def delete_container(container: Container, is_force:bool = False): ''' rm a container which not running like command 'docker rm' args: container (Container): container object ''' - container.remove() + container.remove(force = is_force) @staticmethod def start_container(container: Container): @@ -203,67 +207,17 @@ class DockerProxy: return res - def container_run_command(self, - image:str, - command: str, - user: str, - volumes: list, - work_space: str): - ''' - run command like 'docker run' with tty being true - to keep container alive and then run command in - docker container - ''' - container = self._docker.containers.run( - image=image, - command="bash", - volumes=volumes, - detach=True, - tty=True - ) - if isinstance(container, Container): - res = self.container_exec_command( - container=container, - command=command, - user=user, - work_space=work_space) - return container, res.output - else: - raise ValueError("docker start faild") - - def container_run_simple(self, image:str, volumes: list, network="host", is_priv=False): - ''' - it's just create a tty docker container to do some thing next - ''' - container = self._docker.containers.run( - image=image, - command="bash", - volumes=volumes, - network_mode=network, - detach=True, - tty=True, - privileged=is_priv - ) - container_name = str(container.attrs.get('Name')).lstrip("/") - container.rename(f"oebuild_{container_name}") - return container - - def container_exec_with_tty(self, - container: Container, - user:str, - work_space: str): + def create_container(self, image:str, parameters:str, volumes:list[str], command:str) -> str: ''' - run docker container with tty, you can has a tty terminal + create a new container ''' - cli = container.exec_run( - cmd="bash", - stdout=True, - stderr=True, - stdin=True, - user=user, - workdir=work_space, - tty=True, - socket=True, - ).output - - return cli.fileno() + run_command = f"docker run {parameters}" + for volume in volumes: + run_command = f"{run_command} -v {volume}" + run_command = f"{run_command} {image} {command}" + res = subprocess.run(run_command, shell=True, capture_output=True, check=True, text=True) + if res.returncode != 0: + logger.error(res.stderr.strip()) + sys.exit(res.returncode) + container_id = res.stdout.strip() + return self.get_container(container_id=container_id) diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index 0aa2238..8b05595 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -15,9 +15,9 @@ import re import sys from oebuild.parse_compile import ParseCompile -from oebuild.parse_template import BUILD_IN_DOCKER, BUILD_IN_HOST import oebuild.util as oebuild_util from oebuild.m_log import logger +import oebuild.const as oebuild_const class BaseLocalConf(ValueError): ''' @@ -32,16 +32,7 @@ class NativesdkNotValid(BaseLocalConf): nativesdk directory not valid ''' -NATIVESDK_DIR_NAME = "OPENEULER_NATIVESDK_SYSROOT" -OPENEULER_SP_DIR = "OPENEULER_SP_DIR" -SSTATE_MIRRORS = "SSTATE_MIRRORS" -SSTATE_DIR = "SSTATE_DIR" -TMP_DIR = "TMPDIR" - -NATIVE_GCC_DIR = '/usr1/openeuler/native_gcc' -SSTATE_CACHE = '/usr1/openeuler/sstate-cache' - -def get_nativesdk_sysroot(nativesdk_dir = oebuild_util.NATIVESDK_DIR): +def get_nativesdk_sysroot(nativesdk_dir = oebuild_const.NATIVESDK_DIR): ''' return environment initialization shell, if nativesdk directory is not exists or can not find any initialization shell, raise error @@ -132,8 +123,8 @@ class LocalConf: # replace toolchain if parse_compile.toolchain_dir is not None: - if parse_compile.build_in == BUILD_IN_DOCKER: - replace_toolchain_str = f'{parse_compile.toolchain_type} = "{NATIVE_GCC_DIR}"' + if parse_compile.build_in == oebuild_const.BUILD_IN_DOCKER: + replace_toolchain_str = f'{parse_compile.toolchain_type} = "{oebuild_const.NATIVE_GCC_DIR}"' else: replace_toolchain_str = f'{parse_compile.toolchain_type} = "{parse_compile.toolchain_dir}"' content = match_and_replace( @@ -143,21 +134,21 @@ class LocalConf: ) # replace nativesdk OPENEULER_SP_DIR - if parse_compile.build_in == BUILD_IN_HOST: + if parse_compile.build_in == oebuild_const.BUILD_IN_HOST: self.check_nativesdk_valid(parse_compile.nativesdk_dir) if parse_compile.nativesdk_dir is None: raise ValueError("please set nativesdk dir") nativesdk_sysroot = get_nativesdk_sysroot(parse_compile.nativesdk_dir) nativesdk_sys_dir = os.path.join(parse_compile.nativesdk_dir, nativesdk_sysroot) content = match_and_replace( - pre=NATIVESDK_DIR_NAME, - new_str=f'{NATIVESDK_DIR_NAME} = "{nativesdk_sys_dir}"', + pre=oebuild_const.NATIVESDK_DIR_NAME, + new_str=f'{oebuild_const.NATIVESDK_DIR_NAME} = "{nativesdk_sys_dir}"', content=content ) content = match_and_replace( - pre=OPENEULER_SP_DIR, - new_str= f"{OPENEULER_SP_DIR} = '{src_dir}'", + pre=oebuild_const.OPENEULER_SP_DIR, + new_str= f"{oebuild_const.OPENEULER_SP_DIR} = '{src_dir}'", content=content ) @@ -166,29 +157,29 @@ class LocalConf: if os.path.islink(parse_compile.sstate_cache): new_str= f"file://.* {parse_compile.sstate_cache}/PATH;downloadfilename=PATH" else: - if parse_compile.build_in == BUILD_IN_DOCKER: - new_str= f"file://.* file://{SSTATE_CACHE}/PATH" + if parse_compile.build_in == oebuild_const.BUILD_IN_DOCKER: + new_str= f"file://.* file://{oebuild_const.SSTATE_CACHE}/PATH" else: new_str= f"file://.* file://{parse_compile.sstate_cache}/PATH" content = match_and_replace( - pre=SSTATE_MIRRORS, - new_str = f'{SSTATE_MIRRORS} = "{new_str}"', + pre=oebuild_const.SSTATE_MIRRORS, + new_str = f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"', content=content ) # replace sstate_dir if parse_compile.sstate_dir is not None: content = match_and_replace( - pre=SSTATE_DIR, - new_str = f'{SSTATE_DIR} = "{parse_compile.sstate_dir}"', + pre=oebuild_const.SSTATE_DIR, + new_str = f'{oebuild_const.SSTATE_DIR} = "{parse_compile.sstate_dir}"', content=content ) # replace tmpdir if parse_compile.tmp_dir is not None: content = match_and_replace( - pre=TMP_DIR, - new_str = f'{TMP_DIR} = "{parse_compile.tmp_dir}"', + pre=oebuild_const.TMP_DIR, + new_str = f'{oebuild_const.TMP_DIR} = "{parse_compile.tmp_dir}"', content=content ) diff --git a/src/oebuild/parse_compile.py b/src/oebuild/parse_compile.py index 9023d67..a1c79b5 100644 --- a/src/oebuild/parse_compile.py +++ b/src/oebuild/parse_compile.py @@ -14,10 +14,28 @@ import os from dataclasses import dataclass import pathlib from typing import Optional +import sys import oebuild.util as oebuild_util from oebuild.ogit import OGit -from oebuild.parse_template import PlatformTemplate, ParseTemplate, BUILD_IN_DOCKER +from oebuild.parse_template import PlatformTemplate, ParseTemplate +import oebuild.const as oebuild_const +from oebuild.m_log import logger + +@dataclass +class DockerParam: + ''' + xxxxxxxxx + ''' + # point out the docker image + image: str + # point out the parameter for create container + parameters: list[str] + # point out the volumns for create container + volumns: list[str] + # point out the command for create container + command: str + @dataclass class Compile(PlatformTemplate): @@ -38,7 +56,7 @@ class Compile(PlatformTemplate): tmp_dir: Optional[str] - docker_image: Optional[str] + docker_param: Optional[DockerParam] class BaseParseCompileError(ValueError): ''' @@ -71,9 +89,22 @@ class ParseCompile: except Exception as e_p: raise e_p - default_image = "swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container:latest" + docker_param:DockerParam = None + if "docker_param" in data and data['docker_param'] is not None: + try: + dparam = data['docker_param'] + docker_param = DockerParam( + image=dparam['image'], + parameters=dparam['parameters'], + volumns=dparam['volumns'], + command=dparam['command'] + ) + except KeyError: + logger.error("the compile.yaml parse failed, please check it again") + sys.exit(1) + self.compile = Compile( - build_in=BUILD_IN_DOCKER if 'build_in' not in data else data['build_in'], + build_in=oebuild_const.BUILD_IN_DOCKER if 'build_in' not in data else data['build_in'], platform=data['platform'], machine=data['machine'], toolchain_type=data['toolchain_type'], @@ -86,8 +117,7 @@ class ParseCompile: repos=None if "repos" not in data else ParseTemplate.parse_oebuild_repo(data['repos']), local_conf=None if "local_conf" not in data else data['local_conf'], layers=None if "layers" not in data else data['layers'], - docker_image=default_image if "docker_image" not in data else data['docker_image'] - ) + docker_param=docker_param) @property def build_in(self): @@ -179,13 +209,13 @@ class ParseCompile: return attr of repos ''' return self.compile.repos - + @property - def docker_image(self): + def docker_param(self): ''' - return attr of docker_tag + return attr of docker_param ''' - return self.compile.docker_image + return self.compile.docker_param def check_with_version(self, base_dir, manifest_path): ''' diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 4443d27..1eaecf8 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -18,10 +18,7 @@ import os from ruamel.yaml.scalarstring import LiteralScalarString import oebuild.util as oebuild_util - -PLATFORM = 'platform' -BUILD_IN_DOCKER = "docker" -BUILD_IN_HOST = "host" +import oebuild.const as oebuild_const @dataclass class OebuildRepo: @@ -130,7 +127,7 @@ class ParseTemplate: config_type = data['type'] config_name = os.path.basename(config_dir) - if config_type == PLATFORM: + if config_type == oebuild_const.PLATFORM: self.platform_template = PlatformTemplate( platform=LiteralScalarString(os.path.splitext(config_name)[0]), machine=data['machine'], @@ -162,15 +159,17 @@ class ParseTemplate: except Exception as e_p: raise e_p - def generate_template(self, + def generate_compile_conf(self, nativesdk_dir = None, toolchain_dir = None, - build_in: str = BUILD_IN_DOCKER, + build_in: str = oebuild_const.BUILD_IN_DOCKER, sstate_cache = None, tmp_dir = None, datetime = None, is_disable_fetch = False, - docker_image = ""): + docker_image: str = None, + src_dir: str = None, + compile_dir: str = None): ''' first param common yaml ''' @@ -183,7 +182,6 @@ class ParseTemplate: common_yaml_dir = pathlib.Path(common_yaml_dir) data = oebuild_util.read_yaml(common_yaml_dir) - data['docker_image'] = docker_image repos = {} if 'repos' in data : @@ -237,12 +235,11 @@ class ParseTemplate: disable_fetch_str = LiteralScalarString('OPENEULER_FETCH = "disable"') local_conf = LiteralScalarString(local_conf + '\n' + disable_fetch_str) - compile_conf = { - 'build_in': build_in, - 'docker_image': docker_image, - 'platform': self.platform_template.platform, - 'machine': self.platform_template.machine, - 'toolchain_type': self.platform_template.toolchain_type} + compile_conf = {} + compile_conf['build_in'] = build_in + compile_conf['platform'] = self.platform_template.platform + compile_conf['machine'] = self.platform_template.machine + compile_conf['toolchain_type'] = self.platform_template.toolchain_type if nativesdk_dir is not None: compile_conf['nativesdk_dir'] = nativesdk_dir @@ -257,6 +254,27 @@ class ParseTemplate: compile_conf['local_conf'] = local_conf compile_conf['layers'] = layers + if build_in == oebuild_const.BUILD_IN_HOST: + return compile_conf + + parameters = ['-itd'] + volumns = [] + volumns.append("/dev/net/tun:/dev/net/tun") + volumns.append(src_dir + ':' + oebuild_const.CONTAINER_SRC) + volumns.append(compile_dir + ':' + + os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) + if toolchain_dir is not None: + volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) + if sstate_cache is not None: + volumns.append(sstate_cache + ":" + oebuild_const.SSTATE_CACHE) + + docker_param = {} + docker_param['image'] = docker_image + docker_param['parameters'] = parameters + docker_param['volumns'] = volumns + docker_param['command'] = "bash" + compile_conf['docker_param'] = docker_param + return compile_conf @staticmethod diff --git a/src/oebuild/util.py b/src/oebuild/util.py index 27a7d35..dd6c6b6 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -21,25 +21,16 @@ import subprocess from ruamel.yaml import YAML from docker.errors import DockerException - from docker.models.containers import Container + from oebuild.docker_proxy import DockerProxy from oebuild.m_log import logger from oebuild.version import __version__ +import oebuild.const as oebuild_const -CONFIG_YAML = 'config.yaml' -UPGRADE_YAML = 'upgrade.yaml' -COMPILE_YAML = 'compile.yaml.sample' -BASH_END_FLAG = " ###!!!###" -CONTAINER_USER = "openeuler" -CONTAINER_BUILD = '/home/openeuler/build' -DEFAULT_DOCKER = "swr.cn-north-4.myhuaweicloud.com/openeuler-embedded/openeuler-container:latest" -CONTAINER_SRC = '/usr1/openeuler/src' -CONTAINER_USER = "openeuler" -NATIVESDK_DIR = "/opt/buildtools/nativesdk" -PROXY_LIST = ['http_proxy', 'https_proxy'] -def get_nativesdk_environment(nativesdk_dir=NATIVESDK_DIR, container: Container = None): +def get_nativesdk_environment(nativesdk_dir=oebuild_const.NATIVESDK_DIR, + container: Container = None): ''' return environment initialization shell, if nativesdk directory is not exists or can not find any initialization shell, raise error @@ -57,7 +48,7 @@ def get_nativesdk_environment(nativesdk_dir=NATIVESDK_DIR, container: Container if os.path.isfile(abs_path) and not os.path.islink(abs_path): return item else: - res = container.exec_run("ls -al", user=CONTAINER_USER, workdir=nativesdk_dir) + res = container.exec_run("ls -al", user=oebuild_const.CONTAINER_USER, workdir=nativesdk_dir) if res.exit_code != 0: logger.error("can not find any nativesdk environment initialization shell") sys.exit(res.exit_code) @@ -131,19 +122,19 @@ def get_config_yaml_dir(): ''' return config yaml dir ''' - return os.path.join(get_base_oebuild(), 'app/conf', CONFIG_YAML) + return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.CONFIG_YAML) def get_compile_yaml_dir(): ''' return compile.yaml.sample yaml dir ''' - return os.path.join(get_base_oebuild(), 'app/conf', COMPILE_YAML) + return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.COMPILE_YAML) def get_upgrade_yaml_dir(): ''' return upgrade yaml dir ''' - return os.path.join(get_base_oebuild(), 'app/conf', UPGRADE_YAML) + return os.path.join(get_base_oebuild(), 'app/conf', oebuild_const.UPGRADE_YAML) def generate_random_str(randomlength=16): ''' @@ -196,7 +187,7 @@ def restore_bashrc_content(old_content): new_content = '' for line in old_content.split('\n'): line: str = line - if line.endswith(BASH_END_FLAG) or line.replace(" ", '') == '': + if line.endswith(oebuild_const.BASH_END_FLAG) or line.replace(" ", '') == '': continue new_content = new_content + line + '\n' return new_content @@ -208,7 +199,7 @@ def init_bashrc_content(old_content, init_command: list): new_content = restore_bashrc_content(old_content=old_content) for command in init_command: - new_content = new_content + command + BASH_END_FLAG + '\n' + new_content = new_content + command + oebuild_const.BASH_END_FLAG + '\n' return new_content @@ -218,7 +209,7 @@ def add_bashrc(content: str, line: str): ''' if not content.endswith('\n'): content = content + '\n' - content = content + line + BASH_END_FLAG + '\n' + content = content + line + oebuild_const.BASH_END_FLAG + '\n' return content -- Gitee From 1371dd01c0d9a20ee0c89ecca72db8bb640ee47c Mon Sep 17 00:00:00 2001 From: lixinyu Date: Thu, 11 Jan 2024 21:00:21 +0800 Subject: [PATCH 2/2] bitbake: fix the bug that new compile change bring up * when we make a big optimize about start docker. we make a bit change in compile.yaml, so that oebuild can not parse the old compile.yaml, now fix it. Signed-off-by: lixinyu --- .../app/plugins/bitbake/in_container.py | 41 +++++++++++++++++-- src/oebuild/docker_proxy.py | 2 +- src/oebuild/parse_compile.py | 10 +++++ 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index dd88ade..4dd229a 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -48,17 +48,52 @@ class InContainer(BaseBuild): execute bitbake command ''' logger.info("Bitbake starting ...") + + docker_param:DockerParam = None + if parse_compile.docker_param is not None: + docker_param = parse_compile.docker_param + else: + docker_param = self._trans_docker_param( + docker_image=parse_compile.docker_image, + toolchain_dir=parse_compile.toolchain_dir, + sstate_cache=parse_compile.sstate_cache) + # check docker image if exists docker_proxy = DockerProxy() - if not docker_proxy.is_image_exists(parse_compile.docker_param.image): + if not docker_proxy.is_image_exists(docker_param.image): logger.error('''The docker image does not exists, please run fellow command: `oebuild update docker`''') return - self.deal_env_container(env=parse_env,docker_param=parse_compile.docker_param) - + self.deal_env_container(env=parse_env,docker_param=docker_param) self.exec_compile(parse_compile=parse_compile, command=command) + def _trans_docker_param(self, + docker_image: str, + toolchain_dir: str = None, + sstate_cache: str = None) -> DockerParam: + ''' + this function is to adapt the old compile.yaml + ''' + parameters = ['-itd'] + volumns = [] + volumns.append("/dev/net/tun:/dev/net/tun") + volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) + volumns.append(os.getcwd() + ':' + + os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()))) + if toolchain_dir is not None: + volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) + if sstate_cache is not None: + volumns.append(sstate_cache + ":" + oebuild_const.SSTATE_CACHE) + + docker_param = DockerParam( + image=docker_image, + parameters=parameters, + volumns=volumns, + command="bash" + ) + return docker_param + def deal_env_container(self, env: ParseEnv, docker_param: DockerParam): ''' This operation realizes the processing of the container, diff --git a/src/oebuild/docker_proxy.py b/src/oebuild/docker_proxy.py index 9ceba04..077299b 100644 --- a/src/oebuild/docker_proxy.py +++ b/src/oebuild/docker_proxy.py @@ -207,7 +207,7 @@ class DockerProxy: return res - def create_container(self, image:str, parameters:str, volumes:list[str], command:str) -> str: + def create_container(self, image:str, parameters:str, volumes:list[str], command:str) -> Container: ''' create a new container ''' diff --git a/src/oebuild/parse_compile.py b/src/oebuild/parse_compile.py index a1c79b5..e943918 100644 --- a/src/oebuild/parse_compile.py +++ b/src/oebuild/parse_compile.py @@ -56,6 +56,8 @@ class Compile(PlatformTemplate): tmp_dir: Optional[str] + docker_image: Optional[str] + docker_param: Optional[DockerParam] class BaseParseCompileError(ValueError): @@ -117,6 +119,7 @@ class ParseCompile: repos=None if "repos" not in data else ParseTemplate.parse_oebuild_repo(data['repos']), local_conf=None if "local_conf" not in data else data['local_conf'], layers=None if "layers" not in data else data['layers'], + docker_image=None if "docker_image" not in data else data['docker_image'], docker_param=docker_param) @property @@ -210,6 +213,13 @@ class ParseCompile: ''' return self.compile.repos + @property + def docker_image(self): + ''' + return attr of repos, it will be abandoned in new version + ''' + return self.compile.docker_image + @property def docker_param(self): ''' -- Gitee