From 6d563d88763d4e8662dd99147d030d5ad6590e4a Mon Sep 17 00:00:00 2001 From: lubinglun Date: Fri, 14 Oct 2022 17:24:03 +0800 Subject: [PATCH 1/2] preloader and loader Signed-off-by: lubinglun Change-Id: I4bc170c1139f09a7990f3347e762acc9fa42d40e --- hb_new/services/loader.py | 129 ++++--- hb_new/services/preloader.py | 53 +-- hb_new/util/loader/load_ohos_build.py | 2 + hb_new/util/loader/subsystem_scan.py | 2 + .../util/preloader/preloader_process_data.py | 324 ++++++++---------- 5 files changed, 248 insertions(+), 262 deletions(-) diff --git a/hb_new/services/loader.py b/hb_new/services/loader.py index fabb30d..32119ec 100644 --- a/hb_new/services/loader.py +++ b/hb_new/services/loader.py @@ -25,7 +25,7 @@ from util.loader import generate_targets_gn # noqa: E402 from util.loader import load_ohos_build # noqa: E402 from util.loader import subsystem_scan # noqa: E402 from scripts.util.file_utils import read_json_file, write_json_file, write_file # noqa: E402, E501 - +from util.logUtil import LogUtil class OHOSLoader(LoadInterface): @@ -57,18 +57,12 @@ class OHOSLoader(LoadInterface): def __post_init__(self): self.source_root_dir = self.config.root_path + '/' - self.gn_root_out_dir = self.config.out_path - if self.gn_root_out_dir.startswith('/'): - self.gn_root_out_dir = os.path.relpath(self.gn_root_out_dir, - self.config.root_path) - + self.gn_root_out_dir = self.config.out_path if not self.config.out_path.startswith('/') else os.path.relpath(self.config.out_path, self.config.root_path) self.os_level = self.config.os_level if self.config.os_level else "standard" self.target_cpu = self.config.target_cpu if self.config.target_cpu else "arm" self.target_os = self.config.target_os if self.config.target_os else "ohos" - self.config_output_relpath = os.path.join( - self.gn_root_out_dir, 'build_configs') - self.config_output_dir = os.path.join( - self.source_root_dir, self.config_output_relpath) + self.config_output_relpath = os.path.join(self.gn_root_out_dir, 'build_configs') + self.config_output_dir = os.path.join(self.source_root_dir, self.config_output_relpath) self.target_arch = '{}_{}'.format(self.target_os, self.target_cpu) self.subsystem_config_file = os.path.join( self.config.root_path, 'out/preloader', self.config.product, 'subsystem_config.json') @@ -76,7 +70,6 @@ class OHOSLoader(LoadInterface): self.config.root_path, 'out/preloader', self.config.product, 'platforms.build') self.exclusion_modules_config_file = os.path.join( self.config.root_path, 'out/preloader', self.config.product, 'exclusion_modules.json') - self.example_subsystem_file = os.path.join( self.config.root_path, 'build', 'subsystem_config_example.json') @@ -142,8 +135,12 @@ class OHOSLoader(LoadInterface): def _internel_run(self): + LogUtil.hb_info('Loading configuration file...') + self.__post_init__() + self._execute_loader_args_display() + self._check_parts_config_info() self._generate_subsystem_configs() @@ -159,12 +156,12 @@ class OHOSLoader(LoadInterface): self._generate_target_gn() - self._generate_phony_targets() + self._generate_phony_targets_build_file() - self._generate_build_targets_info() + self._generate_required_parts_targets() # required_parts_targets_list.json - self._generate_build_target_list() + self._generate_required_parts_targets_list() # parts src flag file self._generate_src_flag() @@ -188,13 +185,14 @@ class OHOSLoader(LoadInterface): # generate syscap self._generate_syscap_files() + LogUtil.write_log(self.config.log_path, 'build configs generation is complete', 'info') # check method '''Description: Check the parameters passed in config. If the parameters are not specified or the file content pointed to by the parameters does not exist, an exception will be thrown directly. - @parameter:self + @parameter:none @return :none ''' @@ -257,11 +255,13 @@ class OHOSLoader(LoadInterface): # generate method - '''Description: Generate SystemCapability.json & syscap.json & syscap.para - @parameter:parts config info, target platform parts, pre syscap info path, system path + '''Description: Generate SystemCapability.json & syscap.json & syscap.para, dir:[ + (//out/preloader/${product_name}/system/etc/SystemCapability.json), + (//out/preloader/rk3568/system/etc/syscap.json), + (//out/preloader/rk3568/system/etc/param/syscap.para)] + @parameter:none @return :none ''' - def _generate_syscap_files(self): pre_syscap_info_path = os.path.dirname(self.platforms_config_file) system_path = os.path.join(self.source_root_dir, os.path.join( @@ -358,10 +358,9 @@ class OHOSLoader(LoadInterface): f.close() '''Description: output infos for testfwk into a json file(/out/${product_name}/build_configs/infos_for_testfwk.json) - @parameter:config output directory, target platform parts, parts config information + @parameter:none @return :none ''' - def _generate_infos_for_testfwk(self): infos_for_testfwk_file = os.path.join(self.config_output_dir, "infos_for_testfwk.json") @@ -378,10 +377,9 @@ class OHOSLoader(LoadInterface): _output_infos, check_changes=True) '''Description: output all target platform parts into a json file(/out/${product_name}/build_configs/target_platforms_parts.json) - @parameter:config output directory, target platform parts + @parameter:none @return :none ''' - def _generate_target_platform_parts(self): target_platform_parts_file = os.path.join(self.config_output_dir, "target_platforms_parts.json") @@ -390,10 +388,9 @@ class OHOSLoader(LoadInterface): check_changes=True) '''Description: Generate parts differences in different platforms, using phone as base.(/out/${product_name}/build_configs/parts_different_info.json) - @parameter: target platform parts, config output directory + @parameter: none @return :none ''' - def _generate_part_different_info(self): parts_different_info = self._get_parts_by_platform() parts_different_info_file = os.path.join(self.config_output_dir, @@ -403,10 +400,9 @@ class OHOSLoader(LoadInterface): check_changes=True) '''Description: output platforms list into a gni file.(/out/${product_name}/build_configs/platforms_list.gni) - @parameter: build platforms, config output directory - @return :none + @parameter: none + @return: none ''' - def _generate_platforms_list(self): platforms_list_gni_file = os.path.join(self.config_output_dir, "platforms_list.gni") @@ -419,10 +415,9 @@ class OHOSLoader(LoadInterface): write_file(platforms_list_gni_file, '\n'.join(_gni_file_content)) '''Description: output auto install part into a json file.(/out/${product_name}/build_configs/auto_install_parts.json) - @parameter: parts config information, config output directory - @return :none + @parameter: none + @return: none ''' - def _generate_auto_install_part(self): parts_path_info = self.parts_config_info.get("parts_path_info") auto_install_part_list = [] @@ -435,10 +430,9 @@ class OHOSLoader(LoadInterface): write_json_file(auto_install_list_file, auto_install_part_list) '''Description: output src flag into a json file.(/out/${product_name}/build_configs/parts_src_flag.json) - @parameter: parts information, config output directory, required parts targets + @parameter: none @return :none ''' - def _generate_src_flag(self): parts_src_flag_file = os.path.join(self.config_output_dir, "parts_src_flag.json") @@ -447,31 +441,28 @@ class OHOSLoader(LoadInterface): check_changes=True) '''Description: output build target list into a json file.(/out/${product_name}/build_configs/required_parts_targets_list.json) - @parameter: required parts targets, config output directory + @parameter: none @return :none ''' - - def _generate_build_target_list(self): + def _generate_required_parts_targets_list(self): build_targets_list_file = os.path.join(self.config_output_dir, "required_parts_targets_list.json") write_json_file(build_targets_list_file, list(self.required_parts_targets.values())) '''Description: output build target info into a json file.(/out/${product_name}/build_configs/required_parts_targets.json) - @parameter: required parts targets, config output directory - @return :none + @parameter: none + @return: none ''' - - def _generate_build_targets_info(self): + def _generate_required_parts_targets(self): build_targets_info_file = os.path.join(self.config_output_dir, "required_parts_targets.json") write_json_file(build_targets_info_file, self.required_parts_targets) '''Description: output platforms part by src into a json file.(/out/${product_name}/build_configs/platforms_parts_by_src.json) - @parameter: parts targets, source root directopry, config output relpath, target platform parts + @parameter: none @return :none ''' - def _generate_platforms_part_by_src(self): platforms_parts_by_src = self._get_platforms_parts() platforms_parts_by_src_file = os.path.join(self.source_root_dir, @@ -481,14 +472,34 @@ class OHOSLoader(LoadInterface): platforms_parts_by_src, check_changes=True) + '''Description: output system configs info into 4 files:[ + (/out/${product_name}/build_configs/subsystem_info/parts_list.gni), + (/out/${product_name}/build_configs/subsystem_info/inner_kits_list.gni), + (/out/${product_name}/build_configs/subsystem_info/system_kits_list.gni), + (/out/${product_name}/build_configs/subsystem_info/parts_test_list.gni), + (/out/${product_name}/build_configs/subsystem_info/BUILD.gn)] + @parameter: none + @return :none + ''' def _generate_target_gn(self): generate_targets_gn.gen_targets_gn(self.required_parts_targets, self.config_output_dir) - def _generate_phony_targets(self): + '''Description: output phony targets build file.(/out/${product_name}/build_configs/phony_target/BUILD.gn) + @parameter: none + @return :none + ''' + def _generate_phony_targets_build_file(self): generate_targets_gn.gen_phony_targets(self.required_phony_targets, self.config_output_dir) + + '''Description: output system configs info into 4 files:[ + (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/BUILG.gn), + (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/zframework_stub_exists.gni)] + @parameter: none + @return :none + ''' def _generate_stub_targets(self): generate_targets_gn.gen_stub_targets( self.parts_config_info.get('parts_kits_info'), @@ -496,10 +507,9 @@ class OHOSLoader(LoadInterface): self.config_output_dir) '''Description: output system capabilities into a json file.(/out/${product_name}/build_configs/${platform}_system_capabilities.json) - @parameter: parts targets, source root directopry, config output relpath, target platform parts + @parameter: none @return :none ''' - def _generate_system_capabilities(self): for platform in self.build_platforms: platform_parts = self.target_platform_parts.get(platform) @@ -520,6 +530,13 @@ class OHOSLoader(LoadInterface): sorted(platform_capabilities), check_changes=True) + '''Description: output system configs info into three json files:[ + (/out/${product_name}/build_configs/subsystem_info/subsystem_build_config.json), + (/out/${product_name}/build_configs/subsystem_info/src_subsystem_info.json), + (/out/${product_name}/build_configs/subsystem_info/no_src_subsystem_info.json)] + @parameter: none + @return :none + ''' def _generate_subsystem_configs(self): build_config_file = os.path.join(self.config_output_dir, 'subsystem_info', "subsystem_build_config.json") @@ -656,7 +673,6 @@ class OHOSLoader(LoadInterface): def _get_required_build_targets(self) -> dict: required_build_targets = {} - #_parts_list = self._get_required_build_parts_list() for _p_name, _info in self.parts_targets.items(): if _p_name not in self.required_parts_targets_list: continue @@ -665,8 +681,6 @@ class OHOSLoader(LoadInterface): def _get_required_phony_targets(self) -> dict: required_build_targets = {} - #_parts_list = self._get_required_build_parts_list() - for _p_name, _info in self.phony_targets.items(): if _p_name not in self.required_parts_targets_list: continue @@ -717,10 +731,9 @@ class OHOSLoader(LoadInterface): return real_name, original_part_name '''Description: called by _out_infos_for_testfwk, output information by platform - @parameter:part name information, part name information + @parameter:none @return :none ''' - def _output_infos_by_platform(self, part_name_infos, parts_info_dict): required_parts = {} subsystem_infos = {} @@ -742,3 +755,21 @@ class OHOSLoader(LoadInterface): result['subsystem_infos'] = subsystem_infos result['part_infos'] = required_parts return result + + def _execute_loader_args_display(self): + args = [] + args.append('platforms_config_file="{}"'.format(self.platforms_config_file)) + args.append('subsystem_config_file="{}"'.format(self.subsystem_config_file)) + args.append('example_subsystem_file="{}"'.format(self.example_subsystem_file)) + args.append('exclusion_modules_config_file="{}"'.format(self.exclusion_modules_config_file)) + args.append('source_root_dir="{}"'.format(self.source_root_dir)) + args.append('gn_root_out_dir="{}"'.format(self.gn_root_out_dir)) + args.append('build_platform_name={}'.format(self.build_platform_name)) + args.append('build_xts={}'.format(self.build_xts)) + args.append('load_test_config={}'.format(self.load_test_config)) + args.append('target_os={}'.format(self.target_os)) + args.append('target_cpu={}'.format(self.target_cpu)) + args.append('os_level={}'.format(self.os_level)) + args.append('ignore_api_check={}'.format(self.ignore_api_check)) + args.append('scalable_build={}'.format(self.scalable_build)) + LogUtil.write_log(self.config.log_path, 'loader args:{}'.format(args), 'info') \ No newline at end of file diff --git a/hb_new/services/preloader.py b/hb_new/services/preloader.py index ebf9066..c760c93 100644 --- a/hb_new/services/preloader.py +++ b/hb_new/services/preloader.py @@ -28,53 +28,43 @@ class OHOSPreloader(PreloadInterface): def __init__(self, config: Config): super().__init__(config) + self._dirs = None + self._outputs = None + self._product = None + self._all_parts = {} + self._build_vars = {} + self._os_level = "" + self._target_cpu = "" + self._target_os = "" + self._toolchain_label = "" + self._subsystem_info = {} - self._dirs = Dirs(config) + def __post_init__(self): - # preloader_output_dir - self.preloader_output_dir = os.path.join( - config.root_path, 'out/preloader', config.product) + self._dirs = Dirs(self._config) # All kinds of output files os.makedirs(self._dirs.preloader_output_dir, exist_ok=True) - self._outputs = Outputs( - self._dirs.preloader_output_dir) + self._outputs = Outputs(self._dirs.preloader_output_dir) - # Product & device + # Product config self._product = Product( - config.product, self._dirs, config.product_json) - - def __post_init__(self): - + self._config.product, self._dirs, self._config.product_json, self._config) + # product parse self._product._do_parse() - # get device information - self._device = self._product._device - # get all parts self._all_parts = self._product._parts # get build config info self._build_vars = self._product._build_vars - if self._device: - device_info = self._device.get_device_info() - if device_info: - if self._config.target_cpu: - device_info["target_cpu"] = self._config.target_cpu - if self._config.compile_config: - device_info[self._config.compile_config] = True - self._build_vars.update(device_info) - # generate toolchain self._os_level = self._build_vars.get('os_level') self._target_os = self._build_vars.get('target_os') self._target_cpu = self._build_vars.get('target_cpu') - self._toolchain_label = self._get_toolchain_label() - - # add toolchain label - self._build_vars['product_toolchain_label'] = self._toolchain_label + self._toolchain_label = self._build_vars['product_toolchain_label'] # subsystem info self._subsystem_info = self._get_org_subsystem_info() @@ -208,7 +198,7 @@ class OHOSPreloader(PreloadInterface): self._subsystem_info.update( self._product._get_product_specific_subsystem()) self._subsystem_info.update( - self._device.get_device_specific_subsystem()) + self._product.get_device_specific_subsystem()) IoUtil.dump_json_file( self._outputs.subsystem_config_json, self._subsystem_info) @@ -232,10 +222,3 @@ class OHOSPreloader(PreloadInterface): self._dirs.source_root_dir, self._dirs.subsystem_config_json) return subsystem_info - def _get_toolchain_label(self): - if self._os_level == 'mini' or self._os_level == 'small': - toolchain_label = "" - else: - toolchain_label = '//build/toolchain/{0}:{0}_clang_{1}'.format( - self._target_os, self._target_cpu) - return toolchain_label diff --git a/hb_new/util/loader/load_ohos_build.py b/hb_new/util/loader/load_ohos_build.py index 8d90c1f..35f0740 100755 --- a/hb_new/util/loader/load_ohos_build.py +++ b/hb_new/util/loader/load_ohos_build.py @@ -16,6 +16,7 @@ import os import sys from . import load_bundle_file +from util.logUtil import LogUtil sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from scripts.util.file_utils import read_json_file, write_json_file, write_file # noqa: E402, E501 pylint: disable=C0413, E0611 @@ -696,4 +697,5 @@ def get_parts_info(source_root_dir, _output_parts_info(parts_config_dict, os.path.join(source_root_dir, config_output_relpath)) parts_config_dict['syscap_info'] = system_syscap + LogUtil.hb_info('all parts scan completed') return parts_config_dict diff --git a/hb_new/util/loader/subsystem_scan.py b/hb_new/util/loader/subsystem_scan.py index 24bbd67..a6f40e6 100755 --- a/hb_new/util/loader/subsystem_scan.py +++ b/hb_new/util/loader/subsystem_scan.py @@ -19,6 +19,7 @@ import argparse sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from scripts.util.file_utils import read_json_file, write_json_file # noqa: E402 E501 +from util.logUtil import LogUtil _default_subsystem = {"common": "build/common"} @@ -100,6 +101,7 @@ def scan(subsystem_config_file, example_subsystem_file, source_root_dir): 'subsystem': _build_configs, 'no_src_subsystem': no_src_subsystem } + LogUtil.hb_info('subsytem config scan completed') return scan_result diff --git a/hb_new/util/preloader/preloader_process_data.py b/hb_new/util/preloader/preloader_process_data.py index b1babdc..561c03b 100644 --- a/hb_new/util/preloader/preloader_process_data.py +++ b/hb_new/util/preloader/preloader_process_data.py @@ -17,7 +17,6 @@ import os from util.ioUtil import IoUtil from util.preloader.parse_vendor_product_config import get_vendor_parts_list -from containers.status import throw_exception class Outputs: @@ -65,27 +64,22 @@ class Dirs: class Product(): - def __init__(self, product_name, config_dirs, config_json): + def __init__(self, product_name, config_dirs, config_json, ohos_config): self._name = product_name self._dirs = config_dirs - self._device = None self._config = {} self._build_vars = {} self._parts = {} self._syscap_info = {} self._parsed = False + self._device_name = "" + self._device_info = {} self._config_file = config_json + self._ohos_config = ohos_config - def _sanitize(self, config): - if config and self._name != config.get('product_name'): - raise Exception( - "product name configuration incorrect for '{}'".format( - self._name)) - - @throw_exception def _get_base_parts(self, base_config_dir, os_level): system_base_config_file = os.path.join(base_config_dir, - '{}_system.json'.format(os_level)) + '{}_system.json'.format(os_level)) if not os.path.exists(system_base_config_file): raise Exception("product configuration '{}' doesn't exist.".format( system_base_config_file)) @@ -123,29 +117,15 @@ class Product(): def _get_product_specific_subsystem(self): info = {} - self._do_parse() subsystem_name = 'product_{}'.format(self._name) - if self._get_product_build_path(): + product_build_path = self._config.get('product_build_path') + if product_build_path: info[subsystem_name] = { 'name': subsystem_name, - 'path': self._get_product_build_path() + 'path': product_build_path } return info - def _get_product_build_path(self): - return self._config.get('product_build_path') - - def _get_parts_and_build_vars(self): - self._config = IoUtil.read_json_file(self._config_file) - version = self._config.get('version', '3.0') - self._update_parts(self._config, version) - self._update_build_vars(self._config, version) - return self._parts, self._build_vars - - def _get_device(self): - self._do_parse() - return self._device - def _remove_excluded_components(self): items_to_remove = [] for part, val in self._parts.items(): @@ -157,36 +137,143 @@ class Product(): def _do_parse(self): self._config = IoUtil.read_json_file(self._config_file) version = self._config.get('version', '3.0') - product_name = self._config.get('product_name') + self._update_syscap_info(self._config) + self._update_device(self._config, version) + self._update_parts(self._config, version) + self._update_build_vars(self._config, version) + if version == '3.0': + if os.path.dirname(self._config_file) != self._dirs.built_in_product_dir and not hasattr(self._config, 'product_build_path'): + self._config['product_build_path'] = os.path.relpath(os.path.dirname(self._config_file), self._dirs.source_root_dir) + self._remove_excluded_components() + self._parsed = True + + def _update_syscap_info(self, config): + product_name = config.get('product_name') if product_name == None: product_name = "" - os_level = self._config.get('type') + os_level = config.get('type') if os_level == None: os_level = "" - api_version = self._config.get('api_version') + api_version = config.get('api_version') if api_version == None: api_version = 0 - manufacturer_id = self._config.get('manufacturer_id') + manufacturer_id = config.get('manufacturer_id') if manufacturer_id == None: manufacturer_id = 0 - - self._syscap_info = {'product': product_name, 'api_version': api_version, - 'system_type': os_level, 'manufacturer_id': manufacturer_id} + self._syscap_info = {'product':product_name, 'api_version':api_version, + 'system_type':os_level, 'manufacturer_id':manufacturer_id} - self._sanitize(self._config) - self._update_device(self._config, version) - self._update_parts(self._config, version) - self._update_build_vars(self._config, version) - if version == '3.0': - if os.path.dirname(self._config_file) != self._dirs.built_in_product_dir and not hasattr(self._config, - 'product_build_path'): - self._config['product_build_path'] = os.path.relpath(os.path.dirname(self._config_file), - self._dirs.source_root_dir) - self._remove_excluded_components() - self._parsed = True + # Update the _build_vars based on the product configuration in the vendor warehouse + def _update_build_vars(self, config, version): + build_vars = {} + if version == "1.0": + build_vars = {"os_level": 'large'} + else: + if version == "2.0": + build_vars['os_level'] = config.get("type", "standard") + device_name = config.get('product_device') + if device_name: + build_vars['device_name'] = device_name + else: + build_vars['device_name'] = '' + build_vars['product_company'] = config.get('product_company') + else: + build_vars['os_level'] = config.get('type', 'mini') + build_vars['device_name'] = config.get('board') + if config.get('product_company'): + build_vars['product_company'] = config.get('product_company') + elif os.path.dirname(self._config_file) != self._dirs.built_in_product_dir: + relpath = os.path.relpath(self._config_file, self._dirs.vendor_dir) + build_vars['product_company'] = relpath.split('/')[0] + else: + build_vars['product_company'] = config.get('device_company') + build_vars['product_name'] = config.get('product_name') + if 'enable_ramdisk' in config: + build_vars['enable_ramdisk'] = config.get('enable_ramdisk') + if 'build_selinux' in config: + build_vars['build_selinux'] = config.get('build_selinux') + if 'build_seccomp' in config: + build_vars['build_seccomp'] = config.get('build_seccomp') + if 'support_jsapi' in config: + build_vars['support_jsapi'] = config.get('support_jsapi') + build_vars.update(self._device_info) + if build_vars['os_level'] == 'mini' or build_vars['os_level'] == 'small': + toolchain_label = "" + else: + toolchain_label = '//build/toolchain/{0}:{0}_clang_{1}'.format( + self._device_info['target_os'], self._device_info['target_cpu']) + build_vars['product_toolchain_label'] = toolchain_label + self._build_vars = build_vars + + # Update the _device_name and _device_info based on the product configuration in the vendor warehouse + def _update_device(self, config, version): + if version == "2.0": + device_name = config.get('product_device') + if device_name: + self._device_name = device_name + self._device_info = self._get_device_info_v2(device_name, self._dirs.built_in_device_dir) + else: + device_name = config.get('board') + if device_name: + self._device_name = device_name + self._device_info = self._get_device_info_v3(config) + if self._ohos_config.target_cpu: + self._device_info["target_cpu"] = self._ohos_config.target_cpu + if self._ohos_config.compile_config: + self._device_info[self._ohos_config["compile_config"]] = True + + # Update the _parts based on the product configuration in the vendor warehouse + def _update_parts(self, config, version): + if version == "1.0": + _parts = {} + self._parts = _parts + else: + # 1. inherit parts information from base config + if version == "2.0": + os_level = config.get("type", "standard") + else: + os_level = config.get("type", "mini") + # 2. product config based on default minimum system + based_on_mininum_system = config.get('based_on_mininum_system') + if based_on_mininum_system == "true": + self._parts = self._get_base_parts(self._dirs.built_in_base_dir, os_level) + # 3. inherit parts information from inherit config + inherit = config.get('inherit') + if inherit: + self._parts.update( + self._get_inherit_parts(inherit, self._dirs.source_root_dir)) + + # 4. chipset products relate system parts config + sys_info_path = config.get('system_component') + if sys_info_path: + sys_parts = self._get_sys_relate_parts(sys_info_path, self._parts, self._dirs.source_root_dir) + self._parts.update(sys_parts) + all_parts = {} + if version == "2.0": + current_product_parts = config.get("parts") + if current_product_parts: + all_parts.update(current_product_parts) + else: + all_parts.update(get_vendor_parts_list(config)) + all_parts.update(self._get_product_specific_parts()) + + device_name = config.get('board') + if device_name: + all_parts.update(self.get_device_specific_parts()) + self._parts.update(all_parts) + + # Generate build_info needed for V2 configuration + def _get_device_info_v2(self, device_name, config_dir): + device_config_file = os.path.join(config_dir, + '{}.json'.format(device_name)) + device_info = IoUtil.read_json_file(device_config_file) + if device_info and device_info.get('device_name') != device_name: + raise Exception("device name configuration incorrect in '{}'".format( + device_config_file)) + return device_info # Generate build_info needed for V3 configuration - def _get_device_info(self, config): + def _get_device_info_v3(self, config): # NOTE: # Product_name, device_company are necessary for # config.json, DON NOT use .get to replace [] @@ -211,149 +298,30 @@ class Product(): device_info['device_build_path'] = config.get('device_build_path') else: device_build_path = os.path.join(self._dirs.device_dir, - config['device_company'], - config['board']) + config['device_company'], + config['board']) if not os.path.exists(device_build_path): device_build_path = os.path.join(self._dirs.device_dir, - 'board', - config['device_company'], - config['board']) + 'board', + config['device_company'], + config['board']) device_info['device_build_path'] = device_build_path return device_info - # Update the _build_vars based on the product configuration in the vendor warehouse - def _update_build_vars(self, config, version): - build_vars = {} - if version == "1.0": - build_vars = {"os_level": 'large'} - else: - if version == "2.0": - build_vars['os_level'] = config.get("type", "standard") - device_name = config.get('product_device') - if device_name: - build_vars['device_name'] = device_name - else: - build_vars['device_name'] = '' - build_vars['product_company'] = config.get('product_company') - else: - build_vars['os_level'] = config.get('type', 'mini') - build_vars['device_name'] = config.get('board') - if config.get('product_company'): - build_vars['product_company'] = config.get('product_company') - elif os.path.dirname(self._config_file) != self._dirs.built_in_product_dir: - relpath = os.path.relpath(self._config_file, self._dirs.vendor_dir) - build_vars['product_company'] = relpath.split('/')[0] - else: - build_vars['product_company'] = config.get('device_company') - build_vars['product_name'] = config.get('product_name') - if 'enable_ramdisk' in config: - build_vars['enable_ramdisk'] = config.get('enable_ramdisk') - if 'build_selinux' in config: - build_vars['build_selinux'] = config.get('build_selinux') - if 'build_seccomp' in config: - build_vars['build_seccomp'] = config.get('build_seccomp') - if 'support_jsapi' in config: - build_vars['support_jsapi'] = config.get('support_jsapi') - self._build_vars = build_vars - - # Update the _device based on the product configuration in the vendor warehouse - def _update_device(self, config, version): - if version == "2.0": - device_name = config.get('product_device') - if device_name: - self._device = _MyDevice(device_name, self._dirs) - else: - device_name = config.get('board') - if device_name: - device_info = self._get_device_info(config) - self._device = _MyDevice(device_name, self._dirs, device_info) - - # Update the _parts based on the product configuration in the vendor warehouse - def _update_parts(self, config, version): - if version == "1.0": - _parts = {} - self._parts = _parts - else: - # 1. inherit parts information from base config - if version == "2.0": - os_level = config.get("type", "standard") - else: - os_level = config.get("type", "mini") - # 2. product config based on default minimum system - based_on_mininum_system = config.get('based_on_mininum_system') - if based_on_mininum_system == "true": - self._parts = self._get_base_parts(self._dirs.built_in_base_dir, os_level) - # 3. inherit parts information from inherit config - inherit = config.get('inherit') - if inherit: - self._parts.update( - self._get_inherit_parts(inherit, self._dirs.source_root_dir)) - - # 4. chipset products relate system parts config - sys_info_path = config.get('system_component') - if sys_info_path: - sys_parts = self._get_sys_relate_parts(sys_info_path, self._parts, self._dirs.source_root_dir) - self._parts.update(sys_parts) - all_parts = {} - if version == "2.0": - current_product_parts = config.get("parts") - if current_product_parts: - all_parts.update(current_product_parts) - else: - all_parts.update(get_vendor_parts_list(config)) - all_parts.update(self._get_product_specific_parts()) - - device_name = config.get('board') - if device_name: - all_parts.update(self._device.get_device_specific_parts()) - self._parts.update(all_parts) - - -class _MyDevice(): - - def __init__(self, device_name, config_dirs, device_info=None): - self._name = device_name - self._dirs = config_dirs - if device_info is None: - self._device_info = self._make_device_info( - self._name, self._dirs.built_in_device_dir) - else: - self._device_info = device_info - - def get_device_info(self): - return self._device_info - - def _make_device_info(self, device_name, config_dir): - device_config_file = os.path.join(config_dir, - '{}.json'.format(device_name)) - device_info = IoUtil.read_json_file(device_config_file) - if device_info and device_info.get('device_name') != device_name: - raise Exception("device name configuration incorrect in '{}'".format( - device_config_file)) - return device_info - def get_device_specific_parts(self): info = {} - if self._device_info: - device_build_path = self._device_info.get('device_build_path') - if device_build_path: - subsystem_name = 'device_{}'.format(self._name) - part_name = subsystem_name - info['{}:{}'.format(subsystem_name, part_name)] = {} + if self._device_info and self._device_info.get('device_build_path'): + subsystem_name = 'device_{}'.format(self._device_name) + part_name = subsystem_name + info['{}:{}'.format(subsystem_name, part_name)] = {} return info def get_device_specific_subsystem(self): info = {} - subsystem_name = 'device_{}'.format(self._name) - if self._get_device_build_path(): + subsystem_name = 'device_{}'.format(self._device_name) + if self._device_info and self._device_info.get('device_build_path'): info[subsystem_name] = { 'name': subsystem_name, - 'path': self._get_device_build_path() + 'path': self._device_info.get('device_build_path') } return info - - def _get_device_build_path(self): - if self._device_info: - return self._device_info.get('device_build_path') - else: - return None -- Gitee From e0b2fde76e87d10e280e9afb163a1bbee4091f3b Mon Sep 17 00:00:00 2001 From: lubinglun Date: Sun, 16 Oct 2022 23:41:21 +0800 Subject: [PATCH 2/2] fix product parse process Signed-off-by: lubinglun Change-Id: Idb74587e2eeb72a2aa24a0702a5c8f3298f45b64 --- hb_new/services/preloader.py | 24 +- .../util/preloader/preloader_process_data.py | 298 ++++++++++-------- 2 files changed, 162 insertions(+), 160 deletions(-) diff --git a/hb_new/services/preloader.py b/hb_new/services/preloader.py index c760c93..e35ec56 100644 --- a/hb_new/services/preloader.py +++ b/hb_new/services/preloader.py @@ -40,38 +40,20 @@ class OHOSPreloader(PreloadInterface): self._subsystem_info = {} def __post_init__(self): - self._dirs = Dirs(self._config) - - # All kinds of output files os.makedirs(self._dirs.preloader_output_dir, exist_ok=True) self._outputs = Outputs(self._dirs.preloader_output_dir) - - # Product config - self._product = Product( - self._config.product, self._dirs, self._config.product_json, self._config) - - # product parse - self._product._do_parse() - - # get all parts + self._product = Product(self._dirs, self._config) self._all_parts = self._product._parts - - # get build config info self._build_vars = self._product._build_vars - - # generate toolchain self._os_level = self._build_vars.get('os_level') self._target_os = self._build_vars.get('target_os') self._target_cpu = self._build_vars.get('target_cpu') self._toolchain_label = self._build_vars['product_toolchain_label'] - - # subsystem info self._subsystem_info = self._get_org_subsystem_info() def _internel_run(self): self.__post_init__() - self._generate_build_prop() self._generate_build_config_json() self._generate_parts_json() @@ -198,7 +180,7 @@ class OHOSPreloader(PreloadInterface): self._subsystem_info.update( self._product._get_product_specific_subsystem()) self._subsystem_info.update( - self._product.get_device_specific_subsystem()) + self._product._get_device_specific_subsystem()) IoUtil.dump_json_file( self._outputs.subsystem_config_json, self._subsystem_info) @@ -209,7 +191,7 @@ class OHOSPreloader(PreloadInterface): # get method - def _get_org_subsystem_info(self): + def _get_org_subsystem_info(self) -> dict: subsystem_info = {} if self._os_level == "standard": subsystem_info = IoUtil.read_json_file( diff --git a/hb_new/util/preloader/preloader_process_data.py b/hb_new/util/preloader/preloader_process_data.py index 561c03b..ed0effa 100644 --- a/hb_new/util/preloader/preloader_process_data.py +++ b/hb_new/util/preloader/preloader_process_data.py @@ -15,6 +15,7 @@ import os +from resources.config import Config from util.ioUtil import IoUtil from util.preloader.parse_vendor_product_config import get_vendor_parts_list @@ -64,112 +65,121 @@ class Dirs: class Product(): - def __init__(self, product_name, config_dirs, config_json, ohos_config): - self._name = product_name - self._dirs = config_dirs + def __init__(self, config_dirs: Dirs, ohos_config: Config): + self._ohos_config = None + self._dirs = None + self._name = "" self._config = {} self._build_vars = {} self._parts = {} self._syscap_info = {} - self._parsed = False self._device_name = "" self._device_info = {} - self._config_file = config_json - self._ohos_config = ohos_config - - def _get_base_parts(self, base_config_dir, os_level): - system_base_config_file = os.path.join(base_config_dir, - '{}_system.json'.format(os_level)) - if not os.path.exists(system_base_config_file): - raise Exception("product configuration '{}' doesn't exist.".format( - system_base_config_file)) - return IoUtil.read_json_file(system_base_config_file) - - def _get_inherit_parts(self, inherit, source_root_dir): - inherit_parts = {} - for _config in inherit: - _file = os.path.join(source_root_dir, _config) - _info = IoUtil.read_json_file(_file) - parts = _info.get('parts') - if parts: - inherit_parts.update(parts) - else: - inherit_parts.update(get_vendor_parts_list(_info)) - return inherit_parts - - def _get_sys_relate_parts(self, system_component_info, _parts, source_root_dir): - _info = IoUtil.read_json_file(os.path.join(source_root_dir, system_component_info)) - ret = {} - parts = _info.get('parts') - if not parts: - parts = get_vendor_parts_list(_info) - for part, featrue in parts.items(): - if not _parts.get(part): - ret[part] = featrue - return ret - - def _get_product_specific_parts(self): - part_name = 'product_{}'.format(self._name) - subsystem_name = part_name - info = {} - info['{}:{}'.format(subsystem_name, part_name)] = {} - return info - - def _get_product_specific_subsystem(self): - info = {} - subsystem_name = 'product_{}'.format(self._name) - product_build_path = self._config.get('product_build_path') - if product_build_path: - info[subsystem_name] = { - 'name': subsystem_name, - 'path': product_build_path - } - return info + self._config_file = "" + self._version = '' + self.__post_init__(config_dirs, ohos_config) - def _remove_excluded_components(self): - items_to_remove = [] - for part, val in self._parts.items(): - if "exclude" in val and val["exclude"] == "true": - items_to_remove.append(part) - for item in items_to_remove: - del self._parts[item] + def __post_init__(self, config_dirs: Dirs, config: Config): + self._ohos_config = config + self._dirs = config_dirs + self._name = config.product + self._config_file = config.product_json + self._config = self._get_full_product_config() + self._version = self._config.get('version', '3.0') + self._do_parse() + # parse product configuration, then generate parts list and build vars def _do_parse(self): - self._config = IoUtil.read_json_file(self._config_file) - version = self._config.get('version', '3.0') - self._update_syscap_info(self._config) - self._update_device(self._config, version) - self._update_parts(self._config, version) - self._update_build_vars(self._config, version) - if version == '3.0': - if os.path.dirname(self._config_file) != self._dirs.built_in_product_dir and not hasattr(self._config, 'product_build_path'): - self._config['product_build_path'] = os.path.relpath(os.path.dirname(self._config_file), self._dirs.source_root_dir) + self._update_syscap_info() + self._update_device() + self._update_parts() + self._update_build_vars() self._remove_excluded_components() - self._parsed = True - def _update_syscap_info(self, config): - product_name = config.get('product_name') +# update and remove + + # Update the syscap info + def _update_syscap_info(self): + product_name = self._config.get('product_name') if product_name == None: product_name = "" - os_level = config.get('type') + os_level = self._config.get('type') if os_level == None: os_level = "" - api_version = config.get('api_version') + api_version = self._config.get('api_version') if api_version == None: api_version = 0 - manufacturer_id = config.get('manufacturer_id') + manufacturer_id = self._config.get('manufacturer_id') if manufacturer_id == None: manufacturer_id = 0 self._syscap_info = {'product':product_name, 'api_version':api_version, 'system_type':os_level, 'manufacturer_id':manufacturer_id} + # Update the _device_name and _device_info based on the product configuration in the vendor warehouse + def _update_device(self): + if self._version == "2.0": + device_name = self._config.get('product_device') + if device_name: + self._device_name = device_name + self._device_info = self._get_device_info_v2(device_name, self._dirs.built_in_device_dir) + else: + device_name = self._config.get('board') + if device_name: + self._device_name = device_name + self._device_info = self._get_device_info_v3(self._config) + if self._ohos_config.target_cpu: + self._device_info["target_cpu"] = self._ohos_config.target_cpu + if self._ohos_config.compile_config: + self._device_info[self._ohos_config["compile_config"]] = True + + # Update the _parts based on the product configuration in the vendor warehouse + def _update_parts(self): + if self._version == "1.0": + _parts = {} + self._parts = _parts + else: + # 1. inherit parts information from base config + if self._version == "2.0": + os_level = self._config.get("type", "standard") + else: + os_level = self._config.get("type", "mini") + # 2. product config based on default minimum system + based_on_mininum_system = self._config.get('based_on_mininum_system') + if based_on_mininum_system == "true": + self._parts = self._get_base_parts(self._dirs.built_in_base_dir, os_level) + # 3. inherit parts information from inherit config + inherit = self._config.get('inherit') + if inherit: + self._parts.update( + self._get_inherit_parts(inherit, self._dirs.source_root_dir)) + + # 4. chipset products relate system parts config + sys_info_path = self._config.get('system_component') + if sys_info_path: + sys_parts = self._get_sys_relate_parts(sys_info_path, self._parts, self._dirs.source_root_dir) + self._parts.update(sys_parts) + all_parts = {} + if self._version == "2.0": + current_product_parts = self._config.get("parts") + if current_product_parts: + all_parts.update(current_product_parts) + else: + all_parts.update(get_vendor_parts_list(self._config)) + all_parts.update(self._get_product_specific_parts()) + + device_name = self._config.get('board') + if device_name: + all_parts.update(self._get_device_specific_parts()) + self._parts.update(all_parts) + # Update the _build_vars based on the product configuration in the vendor warehouse - def _update_build_vars(self, config, version): + def _update_build_vars(self): + config = self._config build_vars = {} - if version == "1.0": + if self._version == "1.0": build_vars = {"os_level": 'large'} else: - if version == "2.0": + if self._version == "2.0": build_vars['os_level'] = config.get("type", "standard") device_name = config.get('product_device') if device_name: @@ -205,65 +215,19 @@ class Product(): build_vars['product_toolchain_label'] = toolchain_label self._build_vars = build_vars - # Update the _device_name and _device_info based on the product configuration in the vendor warehouse - def _update_device(self, config, version): - if version == "2.0": - device_name = config.get('product_device') - if device_name: - self._device_name = device_name - self._device_info = self._get_device_info_v2(device_name, self._dirs.built_in_device_dir) - else: - device_name = config.get('board') - if device_name: - self._device_name = device_name - self._device_info = self._get_device_info_v3(config) - if self._ohos_config.target_cpu: - self._device_info["target_cpu"] = self._ohos_config.target_cpu - if self._ohos_config.compile_config: - self._device_info[self._ohos_config["compile_config"]] = True - - # Update the _parts based on the product configuration in the vendor warehouse - def _update_parts(self, config, version): - if version == "1.0": - _parts = {} - self._parts = _parts - else: - # 1. inherit parts information from base config - if version == "2.0": - os_level = config.get("type", "standard") - else: - os_level = config.get("type", "mini") - # 2. product config based on default minimum system - based_on_mininum_system = config.get('based_on_mininum_system') - if based_on_mininum_system == "true": - self._parts = self._get_base_parts(self._dirs.built_in_base_dir, os_level) - # 3. inherit parts information from inherit config - inherit = config.get('inherit') - if inherit: - self._parts.update( - self._get_inherit_parts(inherit, self._dirs.source_root_dir)) - - # 4. chipset products relate system parts config - sys_info_path = config.get('system_component') - if sys_info_path: - sys_parts = self._get_sys_relate_parts(sys_info_path, self._parts, self._dirs.source_root_dir) - self._parts.update(sys_parts) - all_parts = {} - if version == "2.0": - current_product_parts = config.get("parts") - if current_product_parts: - all_parts.update(current_product_parts) - else: - all_parts.update(get_vendor_parts_list(config)) - all_parts.update(self._get_product_specific_parts()) + # Remove excluded components + def _remove_excluded_components(self): + items_to_remove = [] + for part, val in self._parts.items(): + if "exclude" in val and val["exclude"] == "true": + items_to_remove.append(part) + for item in items_to_remove: + del self._parts[item] - device_name = config.get('board') - if device_name: - all_parts.update(self.get_device_specific_parts()) - self._parts.update(all_parts) +# get method # Generate build_info needed for V2 configuration - def _get_device_info_v2(self, device_name, config_dir): + def _get_device_info_v2(self, device_name, config_dir) -> dict: device_config_file = os.path.join(config_dir, '{}.json'.format(device_name)) device_info = IoUtil.read_json_file(device_config_file) @@ -273,7 +237,7 @@ class Product(): return device_info # Generate build_info needed for V3 configuration - def _get_device_info_v3(self, config): + def _get_device_info_v3(self, config) -> dict: # NOTE: # Product_name, device_company are necessary for # config.json, DON NOT use .get to replace [] @@ -308,7 +272,7 @@ class Product(): device_info['device_build_path'] = device_build_path return device_info - def get_device_specific_parts(self): + def _get_device_specific_parts(self) -> dict: info = {} if self._device_info and self._device_info.get('device_build_path'): subsystem_name = 'device_{}'.format(self._device_name) @@ -316,7 +280,7 @@ class Product(): info['{}:{}'.format(subsystem_name, part_name)] = {} return info - def get_device_specific_subsystem(self): + def _get_device_specific_subsystem(self) -> dict: info = {} subsystem_name = 'device_{}'.format(self._device_name) if self._device_info and self._device_info.get('device_build_path'): @@ -325,3 +289,59 @@ class Product(): 'path': self._device_info.get('device_build_path') } return info + + def _get_base_parts(self, base_config_dir, os_level) -> dict: + system_base_config_file = os.path.join(base_config_dir, + '{}_system.json'.format(os_level)) + if not os.path.exists(system_base_config_file): + raise Exception("product configuration '{}' doesn't exist.".format( + system_base_config_file)) + return IoUtil.read_json_file(system_base_config_file) + + def _get_inherit_parts(self, inherit, source_root_dir) -> dict: + inherit_parts = {} + for _config in inherit: + _file = os.path.join(source_root_dir, _config) + _info = IoUtil.read_json_file(_file) + parts = _info.get('parts') + if parts: + inherit_parts.update(parts) + else: + inherit_parts.update(get_vendor_parts_list(_info)) + return inherit_parts + + def _get_sys_relate_parts(self, system_component_info, _parts, source_root_dir) -> dict: + _info = IoUtil.read_json_file(os.path.join(source_root_dir, system_component_info)) + ret = {} + parts = _info.get('parts') + if not parts: + parts = get_vendor_parts_list(_info) + for part, featrue in parts.items(): + if not _parts.get(part): + ret[part] = featrue + return ret + + def _get_product_specific_parts(self) -> dict: + part_name = 'product_{}'.format(self._name) + subsystem_name = part_name + info = {} + info['{}:{}'.format(subsystem_name, part_name)] = {} + return info + + def _get_product_specific_subsystem(self) -> dict: + info = {} + subsystem_name = 'product_{}'.format(self._name) + product_build_path = self._config.get('product_build_path') + if product_build_path: + info[subsystem_name] = { + 'name': subsystem_name, + 'path': product_build_path + } + return info + + def _get_full_product_config(self) -> dict: + config = IoUtil.read_json_file(self._config_file) + if config.get("version") == '3.0': + if os.path.dirname(self._config_file) != self._dirs.built_in_product_dir and not hasattr(self._config, 'product_build_path'): + config['product_build_path'] = os.path.relpath(os.path.dirname(self._config_file), self._dirs.source_root_dir) + return config \ No newline at end of file -- Gitee