diff --git a/src/oebuild/app/plugins/bitbake/bitbake.py b/src/oebuild/app/plugins/bitbake/bitbake.py index 30bfe97828e2da13e1ab1bc82484f33887fe9467..4799ffca41253bc89134a903fa8d0bb5f74fec4f 100644 --- a/src/oebuild/app/plugins/bitbake/bitbake.py +++ b/src/oebuild/app/plugins/bitbake/bitbake.py @@ -105,6 +105,11 @@ class Bitbake(OebuildCommand): "yocto-meta-openeuler") manifest_path = os.path.join(yocto_dir, ".oebuild/manifest.yaml") if compile_param.no_layer is None or compile_param.no_layer is not True: + if compile_param.cache_src_dir is not None: + oebuild_util.sync_repo_from_cache( + repo_list=compile_param.repos, + src_dir=self.configure.source_dir(), + cache_src_dir=compile_param.cache_src_dir) oebuild_util.download_repo_from_manifest( repo_list=compile_param.repos, src_dir=self.configure.source_dir(), diff --git a/src/oebuild/app/plugins/bitbake/in_container.py b/src/oebuild/app/plugins/bitbake/in_container.py index 6a59b25aa69fc6289ade3fa838e892ef3159b1da..2bdb72e005dc3cc867d44b7e3a8ff5849dafaa92 100644 --- a/src/oebuild/app/plugins/bitbake/in_container.py +++ b/src/oebuild/app/plugins/bitbake/in_container.py @@ -17,7 +17,7 @@ from docker.models.containers import Container, ExecResult from oebuild.parse_env import ParseEnv from oebuild.docker_proxy import DockerProxy from oebuild.configure import Configure -from oebuild.struct import CompileParam, DockerParam +from oebuild.struct import CompileParam from oebuild.m_log import logger from oebuild.app.plugins.bitbake.base_build import BaseBuild from oebuild.bashrc import Bashrc @@ -56,32 +56,6 @@ class InContainer(BaseBuild): self.bashrc.set_container(container=self.client.get_container(self.container_id)) self.exec_compile(compile_param=compile_param, command=command) - def _trans_docker_param(self, - docker_image: str, - toolchain_dir: str = None, - sstate_mirrors: str = None) -> DockerParam: - ''' - this function is to adapt the old compile.yaml - ''' - parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS - volumns = [] - volumns.append("/dev/net/tun:/dev/net/tun") - volumns.append(self.configure.source_dir() + ':' + oebuild_const.CONTAINER_SRC) - volumns.append(os.getcwd() + ':' + - os.path.join(oebuild_const.CONTAINER_BUILD, os.path.basename(os.getcwd()))) - if toolchain_dir is not None: - volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_MAP) - if sstate_mirrors is not None: - volumns.append(sstate_mirrors + ":" + oebuild_const.SSTATE_MIRRORS) - - docker_param = DockerParam( - image=docker_image, - parameters=parameters, - volumns=volumns, - command="bash" - ) - return docker_param - def exec_compile(self, compile_param: CompileParam, command: str = ""): ''' execute compile task diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index d52f8d32f489fa95edfa02a1cb210a3169edbe79..d10e7daa19f5a652d672a5f254ac92a4e1b583d9 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -53,12 +53,15 @@ class Generate(OebuildCommand): def __init__(self): self.configure = Configure() - self.nativesdk_dir = None - self.toolchain_dir = None - self.llvm_toolchain_dir = None - self.sstate_mirrors = None - self.sstate_dir = None - self.tmp_dir = None + # params is use for record generate command param, had follow params + # nativesdk_dir + # toolchain_dir + # llvm_toolchain_dir + # sstate_mirrors + # sstate_dir + # tmp_dir + # cache_src_dir + self.params = {} self.oebuild_kconfig_path = os.path.expanduser( '~') + '/.local/oebuild_kconfig/' super().__init__('generate', self.help_msg, self.description) @@ -138,19 +141,22 @@ class Generate(OebuildCommand): sys.exit(0) if args.toolchain_dir != '': - self.toolchain_dir = args.toolchain_dir + self.params['toolchain_dir'] = args.toolchain_dir if args.llvm_toolchain_dir != '': - self.llvm_toolchain_dir = args.llvm_toolchain_dir + self.params['llvm_toolchain_dir'] = args.llvm_toolchain_dir if args.sstate_mirrors is not None: - self.sstate_mirrors = args.sstate_mirrors + self.params['sstate_mirrors'] = args.sstate_mirrors if args.sstate_dir is not None: - self.sstate_dir = args.sstate_dir + self.params['sstate_dir'] = args.sstate_dir if args.tmp_dir is not None: - self.tmp_dir = args.tmp_dir + self.params['tmp_dir'] = args.tmp_dir + + if args.cache_src_dir is not None and args.cache_src_dir != '': + self.params['cache_src_dir'] = args.cache_src_dir if args.list: self.list_info() @@ -199,19 +205,20 @@ class Generate(OebuildCommand): out_dir = pathlib.Path(os.path.join(build_dir, 'compile.yaml')) param = parser_template.get_default_generate_compile_conf_param() - param['nativesdk_dir'] = self.nativesdk_dir - param['toolchain_dir'] = self.toolchain_dir - param['llvm_toolchain_dir'] = self.llvm_toolchain_dir + param['nativesdk_dir'] = self.params['nativesdk_dir'] + param['toolchain_dir'] = self.params['toolchain_dir'] + param['llvm_toolchain_dir'] = self.params['llvm_toolchain_dir'] param['build_in'] = args.build_in - param['sstate_mirrors'] = self.sstate_mirrors - param['sstate_dir'] = self.sstate_dir - param['tmp_dir'] = self.tmp_dir + param['sstate_mirrors'] = self.params['sstate_mirrors'] + param['sstate_dir'] = self.params['sstate_dir'] + param['tmp_dir'] = self.params['tmp_dir'] param['datetime'] = args.datetime param['no_fetch'] = args.no_fetch param['no_layer'] = args.no_layer param['docker_image'] = docker_image param['src_dir'] = self.configure.source_dir() param['compile_dir'] = build_dir + param['cache_src_dir'] = self.params['cache_src_dir'] oebuild_util.write_yaml( out_dir, parser_template.generate_compile_conf(param)) @@ -682,6 +689,13 @@ endif default "None" depends on IMAGE """) + # add cache_src_dir directory + common_str += (""" + config COMMON_CACHE_SRC_DIR + string "cache_src_dir (this param is cache src directory)" + default "None" + depends on IMAGE + """) # add build directory common_str += (""" config COMMON_DIRECTORY diff --git a/src/oebuild/app/plugins/generate/parses.py b/src/oebuild/app/plugins/generate/parses.py index 8df8173a330423d738e2aa76166157703a6f5965..029825e472c42395d0ef724346a3965273407709 100644 --- a/src/oebuild/app/plugins/generate/parses.py +++ b/src/oebuild/app/plugins/generate/parses.py @@ -145,6 +145,13 @@ def parsers(parser): This parameter is used to indicate whether to build an SDK ''') + parser.add_argument('--cache_src_dir', + dest='cache_src_dir', + help=''' + if you want to get src repo from a directory otherwise remote, you can set + this parameter + ''') + parser.add_argument('--gcc', dest='gcc', action="store_true", diff --git a/src/oebuild/const.py b/src/oebuild/const.py index 88125720f6c4b362ac023364340b6954202222ca..acd160ed69bfe0dea5c4ab607f85552dd2c40407 100644 --- a/src/oebuild/const.py +++ b/src/oebuild/const.py @@ -37,10 +37,12 @@ NATIVE_GCC_MAP = '/usr1/openeuler/native_gcc' NATIVE_LLVM_MAP = '/usr1/openeuler/native_llvm' SSTATE_MIRRORS_MAP = '/usr1/openeuler/sstate-mirrors' SSTATE_DIR_MAP = '/usr1/openeuler/sstate-dir' +CACHE_SRC_DIR_MAP = '/usr1/openeuler/cache_src' EXTERNAL_LLVM = "EXTERNAL_TOOLCHAIN_LLVM" EXTERNAL_GCC = "EXTERNAL_TOOLCHAIN_GCC" EXTERNAL = "EXTERNAL_TOOLCHAIN" +CACHE_SRC_DIR = "CACHE_SRC_DIR" # used for bitbake/in_container.py BASH_BANNER = ''' diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index 4eed6a2c769fc73f056823ae80a523cad6be919b..0092e46b689991712c2ff2a0f54f786efe842e0d 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -122,14 +122,7 @@ class LocalConf: compile_param.local_conf = f'{pre_content}\n{compile_param.local_conf}' self._add_content_to_local_conf(local_conf=compile_param.local_conf) - def _deal_other_local_param(self, compile_param: CompileParam, src_dir): - pre_content = "" - # add MACHINE - if compile_param.machine is not None: - pre_content += f'MACHINE = "{compile_param.machine}"\n' - - pre_content = self._deal_toolchain_replace(compile_param, pre_content) - + def _deal_llvm_toolchain_dir(self, compile_param: CompileParam): # replace llvm toolchain if compile_param.llvm_toolchain_dir is not None: if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: @@ -139,19 +132,22 @@ class LocalConf: else: replace_toolchain_str = f''' {oebuild_const.EXTERNAL_LLVM} = "{compile_param.llvm_toolchain_dir}"''' - pre_content += replace_toolchain_str + "\n" - - # replace nativesdk OPENEULER_SP_DIR - if compile_param.build_in == oebuild_const.BUILD_IN_HOST: - self.check_nativesdk_valid(compile_param.nativesdk_dir) - if compile_param.nativesdk_dir is None: - raise ValueError("please set nativesdk dir") - nativesdk_sysroot = get_nativesdk_sysroot(compile_param.nativesdk_dir) - nativesdk_sys_dir = os.path.join(compile_param.nativesdk_dir, nativesdk_sysroot) + return replace_toolchain_str + return "" - pre_content += f'{oebuild_const.NATIVESDK_DIR_NAME} = "{nativesdk_sys_dir}"\n' - pre_content += f'{oebuild_const.OPENEULER_SP_DIR} = "{src_dir}"\n' + def _deal_cache_src_dir(self, compile_param: CompileParam): + if compile_param.cache_src_dir is not None: + if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: + # check if exists cache_src_dir, map it to build environment + replace_cache_src_str = f''' +{oebuild_const.CACHE_SRC_DIR} = "{oebuild_const.CACHE_SRC_DIR_MAP}"''' + else: + replace_cache_src_str = f''' +{oebuild_const.CACHE_SRC_DIR} = "{compile_param.cache_src_dir}"''' + return replace_cache_src_str + return "" + def _deal_sstate_mirrors(self, compile_param: CompileParam): # replace sstate_cache if compile_param.sstate_mirrors is not None: if os.path.islink(compile_param.sstate_mirrors): @@ -161,7 +157,33 @@ class LocalConf: new_str = f"file://.* file://{oebuild_const.SSTATE_MIRRORS_MAP}/PATH" else: new_str = f"file://.* file://{compile_param.sstate_mirrors}/PATH" - pre_content += f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"\n' + return f'{oebuild_const.SSTATE_MIRRORS} = "{new_str}"' + return "" + + def _deal_other_local_param(self, compile_param: CompileParam, src_dir): + pre_content = "" + # add MACHINE + if compile_param.machine is not None: + pre_content += f'MACHINE = "{compile_param.machine}"\n' + + pre_content = self._deal_toolchain_replace(compile_param, pre_content) + + pre_content += self._deal_llvm_toolchain_dir(compile_param) + "\n" + + pre_content += self._deal_cache_src_dir(compile_param) + "\n" + + pre_content += self._deal_sstate_mirrors(compile_param) + "\n" + + # replace nativesdk OPENEULER_SP_DIR + if compile_param.build_in == oebuild_const.BUILD_IN_HOST: + self.check_nativesdk_valid(compile_param.nativesdk_dir) + if compile_param.nativesdk_dir is None: + raise ValueError("please set nativesdk dir") + nativesdk_sysroot = get_nativesdk_sysroot(compile_param.nativesdk_dir) + nativesdk_sys_dir = os.path.join(compile_param.nativesdk_dir, nativesdk_sysroot) + + pre_content += f'{oebuild_const.NATIVESDK_DIR_NAME} = "{nativesdk_sys_dir}"\n' + pre_content += f'{oebuild_const.OPENEULER_SP_DIR} = "{src_dir}"\n' # replace sstate_dir if compile_param.sstate_dir is not None: diff --git a/src/oebuild/parse_param.py b/src/oebuild/parse_param.py index e6abb8a1cb8f9181da4401c0e93e8421f3d6d0aa..43cbcacb7cc267bdf721a7b7b1fbf691aa88fda1 100644 --- a/src/oebuild/parse_param.py +++ b/src/oebuild/parse_param.py @@ -134,6 +134,7 @@ class ParseCompileParam: sstate_mirrors=get_value_from_dict('sstate_mirrors', compile_param_dict, None), sstate_dir=get_value_from_dict('sstate_dir', compile_param_dict, None), tmp_dir=get_value_from_dict('tmp_dir', compile_param_dict, None), + cache_src_dir=get_value_from_dict('cache_src_dir', compile_param_dict, None), repos=None if len(repos) == 0 else repos, local_conf=get_value_from_dict('local_conf', compile_param_dict, None), layers=get_value_from_dict('layers', compile_param_dict, None), diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 42e16e82dbdd2a36365c34a8c322a22fa5a062a8..84d477bac9d4c5db9580e96ebadc119ecb495f23 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -194,7 +194,8 @@ class ParseTemplate: no_layer=False, docker_image: str = None, src_dir: str = None, - compile_dir: str = None + compile_dir: str = None, + cache_src_dir: str = None ''' # first param common yaml if self.platform_template is None: @@ -233,6 +234,7 @@ class ParseTemplate: compile_conf['build_in'] = param['build_in'] compile_conf['machine'] = self.platform_template.machine compile_conf['toolchain_type'] = self.platform_template.toolchain_type + compile_conf['cache_src_dir'] = param['cache_src_dir'] compile_conf = self._deal_non_essential_compile_conf_param(param, compile_conf) compile_conf['no_layer'] = param['no_layer'] compile_conf['repos'] = repos @@ -246,6 +248,7 @@ class ParseTemplate: docker_image=param['docker_image'], dir_list={ "src_dir": param['src_dir'], + "cache_src_dir": param['cache_src_dir'], "compile_dir": param['compile_dir'], "toolchain_dir": param['toolchain_dir'], "llvm_toolchain_dir": param['llvm_toolchain_dir'], @@ -312,6 +315,8 @@ def get_docker_param_dict(docker_image, dir_list): volumns.append(dir_list['sstate_mirrors'] + ":" + oebuild_const.SSTATE_MIRRORS) if 'sstate_dir' in dir_list and dir_list['sstate_dir'] is not None: volumns.append(dir_list['sstate_dir'] + ":" + oebuild_const.SSTATE_DIR) + if 'cache_src_dir' in dir_list and dir_list['cache_src_dir'] is not None: + volumns.append(dir_list['cache_src_dir'] + ":" + oebuild_const.CACHE_SRC_DIR_MAP) docker_param = {} docker_param['image'] = docker_image diff --git a/src/oebuild/struct.py b/src/oebuild/struct.py index 7514e7d9706bd3e66ccc789ec8d13cf8c6f3afd0..6d60ce4190d7a0c32a6fc77b83996d5426df3f86 100644 --- a/src/oebuild/struct.py +++ b/src/oebuild/struct.py @@ -50,6 +50,7 @@ class CompileLocalParam: sstate_mirrors: Optional[str] sstate_dir: Optional[str] tmp_dir: Optional[str] + cache_src_dir: Optional[str] @dataclass diff --git a/src/oebuild/util.py b/src/oebuild/util.py index 593f6833d73852f1030f9b74c300948e11221f27..163917b97ca2ee8878145b2be1e551254cd6b8d7 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -298,6 +298,21 @@ def download_repo_from_manifest(repo_list, src_dir, manifest_path): repo_git.check_out_version(version=repo_obj.version) +def sync_repo_from_cache(repo_list, src_dir, cache_src_dir): + ''' + sync repos from src cache if need + ''' + for repo_name in repo_list: + if os.path.exists(f"{src_dir}/{repo_name}"): + continue + subprocess.run(f"mkdir -p {src_dir}/{repo_name}", + shell=True, + check=True) + subprocess.run(f"rsync -a {cache_src_dir}/{repo_name}/ {src_dir}/{repo_name}/", + shell=True, + check=True) + + def get_docker_image_from_yocto(yocto_dir): ''' determine yocto-meta-openeuler's supported docker image in env.yaml