From db03c7bf683b982263f274b650037b90682ac6cb Mon Sep 17 00:00:00 2001 From: alichinese Date: Tue, 7 May 2024 11:57:50 +0800 Subject: [PATCH] toolchain: optimize toolchain build * fix the toolchain build about mkdir directory * optimize toolchain build exprerience Signed-off-by: alichinese --- src/oebuild/app/plugins/generate/generate.py | 33 ++++++-- .../app/plugins/toolchain/toolchain.py | 81 ++++++++++--------- src/oebuild/bashrc.py | 9 +++ src/oebuild/parse_template.py | 8 +- 4 files changed, 84 insertions(+), 47 deletions(-) diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index f02e8da..42fbe7f 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -223,6 +223,7 @@ class Generate(OebuildCommand): if not os.path.exists(config_path): sys.exit(0) generate_command = self.generate_command(config_path) + # sys.exit(0) subprocess.check_output(f'rm -rf {config_path}', shell=True) args = args.parse_args(generate_command) else: @@ -251,7 +252,7 @@ class Generate(OebuildCommand): sys.exit(0) self.build_toolchain(build_dir, toolchain_name_list, auto_build) self._print_toolchain(build_dir=build_dir) - sys.exit(0) + sys.exit(-1) if args.build_in == oebuild_const.BUILD_IN_HOST: try: @@ -455,6 +456,8 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") break if os.path.exists(os.path.join(build_dir, "conf")): rmtree(os.path.join(build_dir, "conf")) + elif os.path.exists(build_dir): + rmtree(build_dir) os.makedirs(build_dir) return build_dir @@ -696,10 +699,10 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") if toolchain_list: for toolchain_info in toolchain_list: - generate_command += ['-tn', toolchain_info.lower()] + generate_command += ['--toolchain_name', toolchain_info.lower()] if auto_build: - generate_command += ['-at'] + generate_command += ['--auto_build'] return generate_command @@ -773,7 +776,7 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") sstate_mirrors=None) config_list = [] for toolchain_name in toolchain_name_list: - if toolchain_name.startwith("config_"): + if toolchain_name.startswith("config_"): config_list.append(toolchain_name) continue config_list.append("config_" + toolchain_name) @@ -785,7 +788,27 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") } ) if auto_build: - subprocess.run('oebuild toolchain auto', shell=True, check=False) + with subprocess.Popen('oebuild toolchain auto', shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=build_dir, + encoding="utf-8", text=True) as s_p: + if s_p.returncode is not None and s_p.returncode != 0: + err_msg = '' + if s_p.stderr is not None: + for line in s_p.stderr: + err_msg.join(line) + raise ValueError(err_msg) + res = None + while res is None: + res = s_p.poll() + if s_p.stdout is not None: + for line in s_p.stdout: + logger.info(line.strip('\n')) + if s_p.stderr is not None: + for line in s_p.stderr: + logger.error(line.strip('\n')) + sys.exit(res) def get_docker_image(yocto_dir, docker_tag, configure: Configure): diff --git a/src/oebuild/app/plugins/toolchain/toolchain.py b/src/oebuild/app/plugins/toolchain/toolchain.py index ecec1fa..b31b102 100644 --- a/src/oebuild/app/plugins/toolchain/toolchain.py +++ b/src/oebuild/app/plugins/toolchain/toolchain.py @@ -48,11 +48,14 @@ class Toolchain(OebuildCommand): super().__init__('toolchain', self.help_msg, self.description) + def __del__(self): + self.bashrc.restore_bashrc() + def do_add_parser(self, parser_adder) -> argparse.ArgumentParser: parser = self._parser(parser_adder, usage=''' - %(prog)s [auto | ] + %(prog)s [auto | prepare| ] ''') @@ -86,6 +89,7 @@ class Toolchain(OebuildCommand): self.client.check_change_ugid( container=self.client.get_container(container_id=self.container_id), container_user=oebuild_const.CONTAINER_USER) + self._set_environment_param() self._set_environment_param() @@ -95,22 +99,16 @@ class Toolchain(OebuildCommand): b_s = f"echo {b_s}" content = self.bashrc.add_bashrc(content=content, line=b_s) self.bashrc.update_bashrc(content=content) + self.bashrc.clean_command_bash() # 调起容器环境 build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) - docker_exec_list = [ - "docker", - "exec", - "-it", - "-u", - oebuild_const.CONTAINER_USER, - "-w", - build_dir, - self.container_id, - "bash" - ] + docker_exec_list = ["docker", "exec", "-it", "-u", oebuild_const.CONTAINER_USER, + "-w", build_dir, self.container_id, "bash"] os.system(" ".join(docker_exec_list)) elif unknown[0] == "auto": self.auto_build(config_list=toolchain_obj.config_list) + elif unknown[0] == "prepare": + self._run_prepare() else: cross_tools_dir = os.path.join( Configure().source_yocto_dir(), ".oebuild/cross-tools/configs") @@ -125,57 +123,62 @@ class Toolchain(OebuildCommand): if config.startswith("config_"): print(config) print("you can run oebuild toolchain aarch64 or oebuild toolchain config_aarch64") + return self._build_toolchain(config_name=config_name) - self.bashrc.restore_bashrc() def auto_build(self, config_list): ''' is for auto build toolchains ''' + self._run_prepare() for config in config_list: self._build_toolchain(config_name=config) def _set_environment_param(self): content = self.bashrc.get_bashrc_content() - open_source_cmd = 'export OPENSOURCE_DIR="$PWD/open_source"' + build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) + open_source_cmd = f'export CROSS_SOURCE="{build_dir}/open_source/"' content = self.bashrc.add_bashrc(content=content, line=open_source_cmd) - x_tools_cmd = 'export X_TOOLS_DIR="$PWD/x-tools"' + x_tools_cmd = f'export CROSS_X_TOOLS="{build_dir}/x-tools/"' content = self.bashrc.add_bashrc(content=content, line=x_tools_cmd) self.bashrc.update_bashrc(content=content) + def _run_prepare(self): + build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) + res: ExecResult = self.client.container_exec_command( + container=self.client.get_container(self.container_id), + command="./cross-tools/prepare.sh ./", + user=oebuild_const.CONTAINER_USER, + params={ + "work_space": build_dir + }) + for line in res.output: + logger.info(line.decode().strip('\n')) + def _build_toolchain(self, config_name): ''' build toolchain with config ''' + build_dir = os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd())) + container = self.client.get_container(self.container_id) + self.client.container_exec_command( + container=container, + command=f"cp {config_name} .config", + user=oebuild_const.CONTAINER_USER, + params={ + "work_space": build_dir, + "stream": False}) content = self.bashrc.get_bashrc_content() - src_config_dir = os.path.join( - oebuild_const.CONTAINER_SRC, - "yocto-meta-openeuler", - ".oebuild", - "cross-tools", - "configs") - content = self.bashrc.add_bashrc(content=content, line="./cross-tools/prepare.sh ./") - content = self.bashrc.add_bashrc( - content=content, - line=f"cp {src_config_dir}/{config_name} .config && ct-ng build") + content = self.bashrc.add_bashrc(content=content, line="ct-ng build") self.bashrc.update_bashrc(content=content) + self.bashrc.clean_command_bash() res: ExecResult = self.client.container_exec_command( - container=self.client.get_container(self.container_id), - command="bash .bashrc", + container=container, + command=f"bash /home/{oebuild_const.CONTAINER_USER}/.bashrc", user=oebuild_const.CONTAINER_USER, - params={ - "work_space": f"/home/{oebuild_const.CONTAINER_USER}", - "demux": True - }) - exit_code = 0 + params={"work_space": build_dir}) for line in res.output: - if line[1] is not None: - logger.error(line[1].decode().strip('\n')) - exit_code = 1 - else: - logger.info(line[0].decode().strip('\n')) - if exit_code != 0: - sys.exit(exit_code) + logger.info(line.decode().strip('\n')) def _check_support_toolchain(self): return os.path.exists(self._toolchain_yaml_path) diff --git a/src/oebuild/bashrc.py b/src/oebuild/bashrc.py index e25c2b4..e9a88d3 100644 --- a/src/oebuild/bashrc.py +++ b/src/oebuild/bashrc.py @@ -62,6 +62,15 @@ mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINE ) os.remove(tmp_file) + def clean_command_bash(self): + ''' + this is for finished .bashrc then clean command auto + ''' + old_content = self.get_bashrc_content() + clean_command = f"sed -i '/{oebuild_const.BASH_END_FLAG.strip()}$/d' ~/.bashrc" + content = Bashrc().add_bashrc(old_content, clean_command) + self.update_bashrc(content=content) + def restore_bashrc(self): ''' Restoring .bashrc will strip out the command line diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 1ff57c8..332801b 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -282,9 +282,11 @@ def get_docker_param_dict(docker_image, src_dir, compile_dir, toolchain_dir, sst parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS volumns = [] volumns.append("/dev/net/tun:/dev/net/tun") - volumns.append(src_dir + ':' + oebuild_const.CONTAINER_SRC) - volumns.append(compile_dir + ":" + os.path.join( - oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) + if src_dir is not None: + volumns.append(src_dir + ':' + oebuild_const.CONTAINER_SRC) + if compile_dir is not None: + volumns.append(compile_dir + ":" + os.path.join( + oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) if toolchain_dir is not None: volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) if sstate_mirrors is not None: -- Gitee