From 38834d96ed1ec62ce7206279bac8160bfa43b296 Mon Sep 17 00:00:00 2001 From: alichinese Date: Thu, 23 May 2024 20:39:42 +0800 Subject: [PATCH] llvm: add support for llvm os build * support to build llvm os in host Signed-off-by: lixinyu --- src/oebuild/app/plugins/bitbake/in_host.py | 3 + src/oebuild/app/plugins/generate/generate.py | 87 ++++++++++---------- src/oebuild/app/plugins/generate/parses.py | 10 ++- src/oebuild/const.py | 5 ++ src/oebuild/local_conf.py | 69 ++++++++++++---- src/oebuild/parse_param.py | 4 + src/oebuild/parse_template.py | 42 ++++++---- src/oebuild/struct.py | 7 +- 8 files changed, 148 insertions(+), 79 deletions(-) diff --git a/src/oebuild/app/plugins/bitbake/in_host.py b/src/oebuild/app/plugins/bitbake/in_host.py index 1a0500f..32f6869 100644 --- a/src/oebuild/app/plugins/bitbake/in_host.py +++ b/src/oebuild/app/plugins/bitbake/in_host.py @@ -38,6 +38,9 @@ class InHost(BaseBuild): ''' execute bitbake commands ''' + # check nativesdk_dir, if not set, use default value + if compile_param.nativesdk_dir is None: + compile_param.nativesdk_dir = oebuild_const.NATIVESDK_DIR self._init_build_sh(build_dir=os.getcwd()) self._mk_build_sh(nativesdk_dir=compile_param.nativesdk_dir, build_dir=os.getcwd()) self.init_bitbake() diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index a92d93d..9a26918 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -55,6 +55,7 @@ class Generate(OebuildCommand): self.configure = Configure() self.nativesdk_dir = None self.toolchain_dir = None + self.llvm_toolchain_dir = None self.sstate_mirrors = None self.tmp_dir = None self.oebuild_kconfig_path = os.path.expanduser( @@ -93,14 +94,11 @@ class Generate(OebuildCommand): if not os.path.exists(config_path): sys.exit(0) g_command = self.generate_command(config_path) - print("===========") - print(g_command) # sys.exit(0) subprocess.check_output(f'rm -rf {config_path}', shell=True) args = args.parse_args(g_command) else: args = args.parse_args(unknown) - build_in = args.build_in auto_build = bool(args.auto_build) if args.nativesdk: @@ -116,7 +114,6 @@ class Generate(OebuildCommand): if args.gcc: # this is for default directory toolchain - print(args.directory) if args.directory is None or args.directory == '': args.directory = "toolchain" toolchain_name_list = args.gcc_name if args.gcc_name else [] @@ -124,7 +121,7 @@ class Generate(OebuildCommand): if build_dir is None: sys.exit(0) self.build_gcc(build_dir, toolchain_name_list, auto_build) - self._print_toolchain(build_dir=build_dir) + self._print_toolchain(build_dir=build_dir, ) sys.exit(0) if args.llvm: @@ -139,18 +136,12 @@ class Generate(OebuildCommand): self._print_toolchain(build_dir=build_dir) sys.exit(0) - if args.build_in == oebuild_const.BUILD_IN_HOST: - try: - self._check_param_in_host(args=args) - except ValueError as v_e: - logger.error(str(v_e)) - sys.exit(-1) - self.nativesdk_dir = args.nativesdk_dir - build_in = oebuild_const.BUILD_IN_HOST - if args.toolchain_dir != '': self.toolchain_dir = args.toolchain_dir + if args.llvm_toolchain_dir != '': + self.llvm_toolchain_dir = args.llvm_toolchain_dir + if args.sstate_mirrors is not None: self.sstate_mirrors = args.sstate_mirrors @@ -206,7 +197,8 @@ class Generate(OebuildCommand): param = parser_template.get_default_generate_compile_conf_param() param['nativesdk_dir'] = self.nativesdk_dir param['toolchain_dir'] = self.toolchain_dir - param['build_in'] = build_in + param['llvm_toolchain_dir'] = self.llvm_toolchain_dir + param['build_in'] = args.build_in param['sstate_mirrors'] = self.sstate_mirrors param['tmp_dir'] = self.tmp_dir param['datetime'] = args.datetime @@ -263,15 +255,6 @@ oebuild toolchain ''' logger.info(format_dir) - def _check_param_in_host(self, args): - if args.toolchain_dir == '': - raise ValueError( - "build in host must points toolchain directory in '-t' param") - - if args.nativesdk_dir == '': - raise ValueError( - "build in host must points nativesdk directory in '-n' param") - def _add_platform_template(self, args, yocto_oebuild_dir, parser_template: ParseTemplate): if args.platform + '.yaml' in os.listdir( @@ -436,7 +419,6 @@ endif with open(kconfig_path, 'w', encoding='utf-8') as kconfig_file: kconfig_file.write(write_data) - print(kconfig_path) kconf = Kconfig(filename=kconfig_path) os.environ["MENUCONFIG_STYLE"] = "aquatic selection=fg:white,bg:blue" with oebuild_util.suppress_print(): @@ -608,7 +590,10 @@ endif """ toolchain_help = ( - "this param is for external toolchain dir, " + "this param is for external gcc toolchain dir, " + "if you want use your own toolchain") + llvm_toolchain_help = ( + "this param is for external llvm toolchain dir, " "if you want use your own toolchain") nativesdk_help = ( "this param is for external nativesdk dir," @@ -664,12 +649,19 @@ endif default "None" depends on IMAGE && BUILD_IN-HOST """) - # add toolchain dir + # add gcc toolchain dir common_str += (f""" config COMMON_TOOLCHAIN-DIR string "toolchain_dir ({toolchain_help})" default "None" - depends on IMAGE && BUILD_IN-HOST + depends on IMAGE + """) + # add llvm toolchain dir + common_str += (f""" + config COMMON_LLVM-TOOLCHAIN-DIR + string "llvm_toolchain_dir ({llvm_toolchain_help})" + default "None" + depends on IMAGE """) # add nativesdk dir common_str += (f""" @@ -706,7 +698,6 @@ endif content = config_file.read() # sys.exit(0) content = re.sub('#.*|.*None.*', "", content) - print(content) common_list = re.findall('(?<=CONFIG_COMMON_).*', content) platform_search = re.search(r"(?<=CONFIG_PLATFORM_).*(?==y)", content) feature_list = re.findall(r"(?<=CONFIG_FEATURE_).*(?==y)", content) @@ -718,8 +709,6 @@ endif llvm_lib = re.search(r"(?<=CONFIG_LLVM-TOOLCHAIN_AARCH64-LIB).*", content) auto_build = re.search(r"(?<=CONFIG_AUTO-BUILD).*(?==y)", content) g_command = [] - print(common_list) - print(build_in) for basic in common_list: basic_info = basic.lower().replace("\"", "").split('=') basic_info[0] = basic_info[0].replace("-", "_") @@ -730,7 +719,6 @@ endif g_command += ['--' + basic_info[0]] continue g_command += ['--' + basic_info[0], basic_info[1]] - print(g_command) # sys.exit(0) if build_in: g_command += ['-b_in', build_in.group().lower()] @@ -792,10 +780,13 @@ endif ) info['docker_param'] = get_docker_param_dict( docker_image=docker_image, - src_dir=self.configure.source_dir(), - compile_dir=compile_dir, - toolchain_dir=None, - sstate_mirrors=None + dir_list={ + "src_dir": self.configure.source_dir(), + "compile_dir": compile_dir, + "toolchain_dir": None, + "llvm_toolchain_dir": None, + "sstate_mirrors": None + } ) # add nativesdk conf nativesdk_yaml_path = os.path.join( @@ -826,10 +817,14 @@ endif # add toolchain.yaml to compile docker_param = get_docker_param_dict( docker_image=get_sdk_docker_image(yocto_dir=self.configure.source_yocto_dir()), - src_dir=self.configure.source_dir(), - compile_dir=build_dir, - toolchain_dir=None, - sstate_mirrors=None) + dir_list={ + "src_dir": self.configure.source_dir(), + "compile_dir": build_dir, + "toolchain_dir": None, + "llvm_toolchain_dir": None, + "sstate_mirrors": None + } + ) config_list = [] for gcc_name in gcc_name_list: if gcc_name.startswith("config_"): @@ -885,10 +880,14 @@ endif # add toolchain.yaml to compile docker_param = get_docker_param_dict( docker_image=get_sdk_docker_image(yocto_dir=self.configure.source_yocto_dir()), - src_dir=self.configure.source_dir(), - compile_dir=build_dir, - toolchain_dir=None, - sstate_mirrors=None) + dir_list={ + "src_dir": self.configure.source_dir(), + "compile_dir": build_dir, + "toolchain_dir": None, + "llvm_toolchain_dir": None, + "sstate_mirrors": None + } + ) oebuild_util.write_yaml( yaml_path=os.path.join(build_dir, 'toolchain.yaml'), data={ diff --git a/src/oebuild/app/plugins/generate/parses.py b/src/oebuild/app/plugins/generate/parses.py index a7d4ce0..31214e9 100644 --- a/src/oebuild/app/plugins/generate/parses.py +++ b/src/oebuild/app/plugins/generate/parses.py @@ -74,7 +74,15 @@ def parsers(parser): dest='toolchain_dir', default='', help=''' - this param is for external toolchain dir, if you want use your own toolchain + this param is for external gcc toolchain dir, if you want use your own toolchain + ''') + + parser.add_argument('-lt', + '--llvm_toolchain_dir', + dest='llvm_toolchain_dir', + default='', + help=''' + this param is for external llvm toolchain dir, if you want use your own toolchain ''') parser.add_argument('-n', diff --git a/src/oebuild/const.py b/src/oebuild/const.py index 42b6282..8517562 100644 --- a/src/oebuild/const.py +++ b/src/oebuild/const.py @@ -34,8 +34,13 @@ SSTATE_DIR = "SSTATE_DIR" TMP_DIR = "TMPDIR" NATIVE_GCC_DIR = '/usr1/openeuler/native_gcc' +NATIVE_LLVM_DIR = '/usr1/openeuler/native_llvm' SSTATE_MIRRORS = '/usr1/openeuler/sstate-cache' +EXTERNAL_LLVM = "EXTERNAL_TOOLCHAIN_LLVM" +EXTERNAL_GCC = "EXTERNAL_TOOLCHAIN_GCC" +EXTERNAL = "EXTERNAL_TOOLCHAIN" + # used for bitbake/in_container.py BASH_BANNER = ''' Welcome to the openEuler Embedded build environment, where you diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index b3de25c..d6b4bb4 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -101,21 +101,20 @@ class LocalConf: def __init__(self, local_conf_path: str): self.local_path = local_conf_path + if not os.path.exists(local_conf_path): + raise ValueError(f'{local_conf_path} not exists') + + with open(local_conf_path, 'r', encoding='utf-8') as r_f: + self.content = r_f.read() def update(self, compile_param: CompileParam, src_dir=None): ''' update local.conf by ParseCompile ''' - if not os.path.exists(self.local_path): - raise ValueError(f'{self.local_path} not exists') - - with open(self.local_path, 'r', encoding='utf-8') as r_f: - content = r_f.read() - pre_content = self._deal_other_local_param(compile_param=compile_param, src_dir=src_dir) compile_param.local_conf = f'{pre_content}\n{compile_param.local_conf}' - self._add_content_to_local_conf(content=content, local_conf=compile_param.local_conf) + self._add_content_to_local_conf(local_conf=compile_param.local_conf) def _deal_other_local_param(self, compile_param: CompileParam, src_dir): pre_content = "" @@ -123,15 +122,17 @@ class LocalConf: if compile_param.machine is not None: pre_content += f'MACHINE = "{compile_param.machine}"\n' - # replace toolchain - if compile_param.toolchain_dir is not None: + pre_content = self._deal_toolchain_replace(compile_param, pre_content) + + # replace llvm toolchain + if compile_param.llvm_toolchain_dir is not None: if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: + # check if exists TXTERNAL_TOOLCHAIN_GCC, replace it or replace TXTERNAL_TOOLCHAIN replace_toolchain_str = f''' -{compile_param.toolchain_type} = "{oebuild_const.NATIVE_GCC_DIR}"''' +{oebuild_const.EXTERNAL_LLVM} = "{oebuild_const.NATIVE_LLVM_DIR}"''' else: replace_toolchain_str = f''' -{compile_param.toolchain_type} = "{compile_param.toolchain_dir}"''' - +{oebuild_const.EXTERNAL_LLVM} = "{compile_param.llvm_toolchain_dir}"''' pre_content += replace_toolchain_str + "\n" # replace nativesdk OPENEULER_SP_DIR @@ -166,21 +167,53 @@ class LocalConf: return pre_content - def _add_content_to_local_conf(self, content, local_conf): + def _deal_toolchain_replace(self, compile_param: CompileParam, pre_content): + # The newly added external compiler chain is named EXTERNAL_TOOLCHAIN_GCC, however, + # the old version still uses EXTERNAL_TOOLCHAIN. Therefore, to maintain compatibility + # with both new and old versions, we have made the following adjustment: + # If EXTERNAL_TOOLCHAIN_GCC exists in the original configuration file, replace + # EXTERNAL_TOOLCHAIN with EXTERNAL_TOOLCHAIN_GCC. + if compile_param.toolchain_dir is not None: + if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: + # check if exists TXTERNAL_TOOLCHAIN_GCC, replace it or replace TXTERNAL_TOOLCHAIN + if len(re.findall(f'^({oebuild_const.EXTERNAL_GCC})*', self.content)) > 0: + toolchain_type = compile_param.toolchain_type.replace( + oebuild_const.EXTERNAL, + oebuild_const.EXTERNAL_GCC) + replace_toolchain_str = f''' +{toolchain_type} = "{oebuild_const.NATIVE_GCC_DIR}"''' + else: + replace_toolchain_str = f''' +{compile_param.toolchain_type} = "{oebuild_const.NATIVE_GCC_DIR}"''' + else: + if len(re.findall(f'^({oebuild_const.EXTERNAL_GCC})*', self.content)) > 0: + toolchain_type = compile_param.toolchain_type.replace( + oebuild_const.EXTERNAL, + oebuild_const.EXTERNAL_GCC) + replace_toolchain_str = f''' +{toolchain_type} = "{compile_param.toolchain_dir}"''' + else: + replace_toolchain_str = f''' +{compile_param.toolchain_type} = "{compile_param.toolchain_dir}"''' + pre_content += replace_toolchain_str + "\n" + + return pre_content + + def _add_content_to_local_conf(self, local_conf): user_content_flag = "#===========the content is user added==================" - if user_content_flag not in content and local_conf is not None and local_conf != "": + if user_content_flag not in self.content and local_conf is not None and local_conf != "": # check if exists remark sysmbol, if exists and replace it - content += f"\n{user_content_flag}\n" + self.content += f"\n{user_content_flag}\n" for line in local_conf.split('\n'): if line.startswith("#"): r_line = line.lstrip("#").strip(" ") - content = content.replace(r_line, line) + self.content = self.content.replace(r_line, line) if line.strip(" ") == "None": continue - content += line + "\n" + self.content += line + "\n" with open(self.local_path, 'w', encoding="utf-8") as r_f: - r_f.write(content) + r_f.write(self.content) def check_nativesdk_valid(self, nativesdk_dir): ''' diff --git a/src/oebuild/parse_param.py b/src/oebuild/parse_param.py index 364c6f4..feae9e8 100644 --- a/src/oebuild/parse_param.py +++ b/src/oebuild/parse_param.py @@ -126,6 +126,7 @@ class ParseCompileParam: tmp_dir: Optional[str] toolchain_dir: Optional[str] + llvm_toolchain_dir: Optional[str] nativesdk_dir: Optional[str] ''' @@ -158,6 +159,7 @@ class ParseCompileParam: machine=get_value_from_dict('machine', compile_param_dict, None), toolchain_type=get_value_from_dict('toolchain_type', compile_param_dict, None), toolchain_dir=get_value_from_dict('toolchain_dir', compile_param_dict, None), + llvm_toolchain_dir=get_value_from_dict('llvm_toolchain_dir', compile_param_dict, None), nativesdk_dir=get_value_from_dict('nativesdk_dir', compile_param_dict, None), sstate_mirrors=get_value_from_dict('sstate_mirrors', compile_param_dict, None), sstate_dir=get_value_from_dict('sstate_dir', compile_param_dict, None), @@ -179,6 +181,8 @@ class ParseCompileParam: compile_obj['toolchain_type'] = compile_param.toolchain_type if compile_param.toolchain_dir is not None: compile_obj['toolchain_dir'] = compile_param.toolchain_dir + if compile_param.llvm_toolchain_dir is not None: + compile_obj['llvm_toolchain_dir'] = compile_param.llvm_toolchain_dir if compile_param.nativesdk_dir is not None: compile_obj['nativesdk_dir'] = compile_param.nativesdk_dir if compile_param.sstate_mirrors is not None: diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 8b7b520..0a468f0 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -186,6 +186,7 @@ class ParseTemplate: param obj: nativesdk_dir=None, toolchain_dir=None, + llvm_toolchain_dir=None build_in: str = oebuild_const.BUILD_IN_DOCKER, sstate_mirrors=None, tmp_dir=None, @@ -244,10 +245,13 @@ class ParseTemplate: compile_conf['docker_param'] = get_docker_param_dict( docker_image=param['docker_image'], - src_dir=param['src_dir'], - compile_dir=param['compile_dir'], - toolchain_dir=param['toolchain_dir'], - sstate_mirrors=param['sstate_mirrors'] + dir_list={ + "src_dir": param['src_dir'], + "compile_dir": param['compile_dir'], + "toolchain_dir": param['toolchain_dir'], + "llvm_toolchain_dir": param['llvm_toolchain_dir'], + "sstate_mirrors": param['sstate_mirrors'] + } ) return compile_conf @@ -257,6 +261,8 @@ class ParseTemplate: compile_conf['nativesdk_dir'] = param['nativesdk_dir'] if param['toolchain_dir'] is not None: compile_conf['toolchain_dir'] = param['toolchain_dir'] + if param['llvm_toolchain_dir'] is not None: + compile_conf['llvm_toolchain_dir'] = param['llvm_toolchain_dir'] if param['sstate_mirrors'] is not None: compile_conf['sstate_mirrors'] = param['sstate_mirrors'] if param['tmp_dir'] is not None: @@ -277,22 +283,30 @@ class ParseTemplate: return repo_cict -def get_docker_param_dict(docker_image, src_dir, compile_dir, toolchain_dir, sstate_mirrors): +def get_docker_param_dict(docker_image, dir_list): ''' transfer docker param to dict + dir_list: + src_dir + compile_dir + toolchain_dir + llvm_toolchain_dir + sstate_mirrors ''' parameters = oebuild_const.DEFAULT_CONTAINER_PARAMS volumns = [] volumns.append("/dev/net/tun:/dev/net/tun") - if src_dir is not None: - volumns.append(src_dir + ':' + oebuild_const.CONTAINER_SRC) - if compile_dir is not None: - volumns.append(compile_dir + ":" + os.path.join( - oebuild_const.CONTAINER_BUILD, os.path.basename(compile_dir))) - if toolchain_dir is not None: - volumns.append(toolchain_dir + ":" + oebuild_const.NATIVE_GCC_DIR) - if sstate_mirrors is not None: - volumns.append(sstate_mirrors + ":" + oebuild_const.SSTATE_MIRRORS) + if dir_list['src_dir'] is not None: + volumns.append(dir_list['src_dir'] + ':' + oebuild_const.CONTAINER_SRC) + if dir_list['compile_dir'] is not None: + volumns.append(dir_list['compile_dir'] + ":" + os.path.join( + oebuild_const.CONTAINER_BUILD, os.path.basename(dir_list['compile_dir']))) + if dir_list['toolchain_dir'] is not None: + volumns.append(dir_list['toolchain_dir'] + ":" + oebuild_const.NATIVE_GCC_DIR) + if dir_list['llvm_toolchain_dir'] is not None: + volumns.append(dir_list['llvm_toolchain_dir'] + ":" + oebuild_const.NATIVE_LLVM_DIR) + if dir_list['sstate_mirrors'] is not None: + volumns.append(dir_list['sstate_mirrors'] + ":" + oebuild_const.SSTATE_MIRRORS) docker_param = {} docker_param['image'] = docker_image diff --git a/src/oebuild/struct.py b/src/oebuild/struct.py index 36e31a8..08e6cd8 100644 --- a/src/oebuild/struct.py +++ b/src/oebuild/struct.py @@ -77,11 +77,14 @@ class CompileParamComm: @dataclass -class CompileParamHost: +class CompileParamSDK: ''' this is parse for host environment ''' + # gcc toolchain dir toolchain_dir: Optional[str] + # llvm toolchain dir + llvm_toolchain_dir: Optional[str] nativesdk_dir: Optional[str] @@ -94,7 +97,7 @@ class CompileParamBitbakeCmds: @dataclass -class CompileParam(CompileParamComm, CompileLocalParam, CompileParamHost, CompileParamBitbakeCmds): +class CompileParam(CompileParamComm, CompileLocalParam, CompileParamSDK, CompileParamBitbakeCmds): ''' Compile is the parsed object of compile.yaml and is used to manipulate the build file ''' -- Gitee