From 5067927a3762dfe17aefc9026c2bcd911533a922 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Wed, 21 Jun 2023 19:59:31 +0800 Subject: [PATCH 1/3] =?UTF-8?q?[rom=5Fram=5Fanalyzer]=E9=80=82=E9=85=8Dfou?= =?UTF-8?q?ndation=E6=8B=89=E8=B5=B7=E8=BF=9B=E7=A8=8B=E7=9A=84=E6=9B=B4?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/standard/README.md | 20 +++--- .../rom_ram_analyzer/standard/ram_analyzer.py | 72 +++++++++---------- 2 files changed, 44 insertions(+), 48 deletions(-) diff --git a/tools/rom_ram_analyzer/standard/README.md b/tools/rom_ram_analyzer/standard/README.md index af157fe..81008bf 100644 --- a/tools/rom_ram_analyzer/standard/README.md +++ b/tools/rom_ram_analyzer/standard/README.md @@ -88,9 +88,9 @@ ## 功能介绍 -基于out/{product_name}/packages/phone下所有cfg文件、out/{product_name}/packages/phone/system/profile下所有xml文件，rom的分析结果，（rom_ram_baseline.json——可以在rom分析阶段通过-b参数生成）分析各进程及对应部件的ram占用（默认取Pss） +基于out/{product_name}/packages/phone下所有cfg文件、out/{product_name}/packages/phone/system/profile下所有json文件，rom的分析结果，（rom_ram_baseline.json——可以在rom分析阶段通过-b参数生成）分析各进程及对应部件的ram占用（默认取Pss） -收集cfg、xml文件的可供参考命令：`mkdir xml && cp $(find ~/oh/out/rk3568/packages/phone/system/profile -name *.xml | xargs) xml` +收集cfg、json文件的可供参考命令：`mkdir cfgs && cp $(find ~/oh/out/rk3568/packages/phone -name *.cfs | xargs) cfgs` 结果以json与xls格式存储，其中，json格式是必输出的，xls格式需要-e参数控制。 @@ -109,7 +109,7 @@ ``` 5. 准备好相关数据： 1. out/{product_name}/packages/phone下所有cfg文件，并将其放置于同一个目录中（ps：同名文件仅保存一份即可） - 1. out/{product_name}/packages/phone/system/profile下所有xml文件 + 1. out/{product_name}/packages/phone/system/profile下所有json 文件 1. rom_ram_baseline.json——如果需要在结果中生成基线信息 6. 运行rom_analyzer.py产生的json结果一份（即-o参数对应的文件，默认rom_analysis_result.json） @@ -118,16 +118,16 @@ 1. 使用`-h`或`--help`查看帮助 ```shell > python .\ram_analyzer.py -h - usage: ram_analyzer.py [-h] [-v] -x XML_PATH -c CFG_PATH [-j ROM_RESULT] -n DEVICE_NUM [-b BASELINE_FILE] - [-o OUTPUT_FILENAME] [-e EXCEL] + usage: ram_analyzer.py [-h] [-v] -s JSON_PATH -c CFG_PATH [-j ROM_RESULT] -n DEVICE_NUM [-b BASELINE_FILE] + [-o OUTPUT_FILENAME] [-u] [-e EXCEL] analyze ram size of component optional arguments: -h, --help show this help message and exit -v, -version show program\'s version number and exit - -x XML_PATH, --xml_path XML_PATH - path of xml file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile + -s JSON_PATH, --json_path JSON_PATH + path of sa json file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile -c CFG_PATH, --cfg_path CFG_PATH path of cfg files. eg: -c ./cfgs/ -j ROM_RESULT, --rom_result ROM_RESULT @@ -139,13 +139,15 @@ baseline file of rom and ram generated by rom analysis. -o OUTPUT_FILENAME, --output_filename OUTPUT_FILENAME base name of output file, default: ram_analysis_result. eg: -o ram_analysis_result + -u, --unit_adaptive unit adaptive -e EXCEL, --excel EXCEL if output result as excel, default: False. eg: -e True ``` 2. 使用示例： ```shell - python ram_analyzer.py -x xml -c cfg -j rom_analysis_result.json -n 7001005458323933328a59a140913900 -b rom_ram_baseline.json -e True - # -x xml：指定xml文件的存放路径为xml目录，-c cfg类似 + python .\ram_analyzer.py -s .\profile\ -c .\cfgs -j .\rom_analysis_result.json -n 7001005458323933328a59a140913900 -e True + # -s profile：copy到本地的out/rk3568/package/phone/system/profile目录 + # -c .\cfgs：copy到本地的out/rk3568/packages/phone下所有cfg # -b rom_ram_baseline.json：指定rom与ram的基线信息文件（可在rom统计阶段使用-b参数生成） # -e True：生成xls格式的结果文件 ``` diff --git a/tools/rom_ram_analyzer/standard/ram_analyzer.py b/tools/rom_ram_analyzer/standard/ram_analyzer.py index eada31c..c6b9fd2 100644 --- a/tools/rom_ram_analyzer/standard/ram_analyzer.py +++ b/tools/rom_ram_analyzer/standard/ram_analyzer.py @@ -172,26 +172,28 @@ class RamAnalyzer: return exec_once() @classmethod - def __parse_process_xml(cls, file_path: str, result_dict: typing.Dict[str, typing.List[str]]): + def __parse_process_json(cls, file_path: str, result_dict: typing.Dict[str, typing.List[str]]): """ - 解析xml文件，结存存入 result_dict中，格式：{process_name: os_list} + 解析json文件，结存存入 result_dict中，格式：{process_name: os_list} 其中，so_list中是so的base_name """ - if not (os.path.isfile(file_path) and file_path.endswith(".xml")): - print("warning: {} not exist or not a xml file".format(file_path)) + if not (os.path.isfile(file_path) and file_path.endswith(".json")): + print("warning: {} not exist or not a json file".format(file_path)) return - doc = dom.parse(file_path) - info = doc.getElementsByTagName("info")[0] - process = info.getElementsByTagName("process")[0] - process_name = process.childNodes[0].data - result_dict[process_name] = list() - libs = info.getElementsByTagName( - "loadlibs")[0].getElementsByTagName("libpath") - for lib in libs: - so = lib.childNodes[0].data - result_dict.get(process_name).append(os.path.split(so)[-1]) - if debug: - print(process_name, " ", so) + with open(file_path, 'r', encoding='utf-8') as f: + j_content:typing.Dict[str, typing.Any] = json.load(f) + if "process" not in j_content.keys() or "systemability" not in j_content.keys(): + print(f"warning: {file_path} has no field 'process' or 'systemability'") + return + process_name:str = j_content.get("process") + elf_list:typing.List[str] = list() + for sa in j_content.get("systemability"): + libpath:str = sa.get("libpath") + if not libpath: + continue + elf_list.append(libpath) + result_dict[process_name] = elf_list + @classmethod def get_elf_info_from_rom_result(cls, rom_result_json: str) -> typing.Dict[str, typing.Dict[str, str]]: @@ -238,7 +240,7 @@ class RamAnalyzer: if first.endswith("sa_main"): # 由sa_main去来起进程 xml_base_name = os.path.split(path_list[0])[-1] - cls.__parse_process_xml(os.path.join( + cls.__parse_process_json(os.path.join( profile_path, xml_base_name), result_dict) else: # 直接执行 @@ -247,25 +249,15 @@ class RamAnalyzer: result_dict.get(process_name).append(os.path.split(first)[-1]) @classmethod - def get_process_so_relationship(cls, xml_path: str, cfg_path: str, profile_path: str) -> typing.Dict[ + def get_process_so_relationship(cls,cfg_path: str, profile_path: str) -> typing.Dict[ str, typing.List[str]]: """ - 从out/{product_name}/packages/phone/sa_profile/merged_sa查找xml文件并处理得到进程与so的对应关系 + parse the relationship between process and elf file """ # 从merged_sa里面收集 - xml_list = glob.glob(xml_path + os.sep + "*[.]xml", recursive=True) + # json_list = glob.glob(json_path + os.sep + "*[.]json", recursive=True) process_elf_dict: typing.Dict[str, typing.List[str]] = dict() - for xml in xml_list: - if debug: - print("parsing: ", xml) - try: - cls.__parse_process_xml(xml, process_elf_dict) - except: - print("parse '{}' failed".format(xml)) - finally: - ... - # 从system/etc/init/*.cfg中收集，如果是sa_main拉起的，则从system/profile/*.xml中进行解析 - cfg_list = glob.glob(cfg_path + os.sep + "*[.]cfg", recursive=True) + cfg_list = glob.glob(cfg_path + os.sep + "*.cfg", recursive=True) for cfg in cfg_list: if debug: print("parsing: ", cfg) @@ -429,7 +421,7 @@ class RamAnalyzer: subsystem_info["size"] = sub_size @classmethod - def analysis(cls, cfg_path: str, xml_path: str, rom_result_json: str, device_num: str, + def analysis(cls, cfg_path: str, json_path: str, rom_result_json: str, device_num: str, output_file: str, ss: str, output_excel: bool, baseline_file: str, unit_adapt:bool): """ process size subsystem/component so so_size @@ -446,8 +438,8 @@ class RamAnalyzer: so_info_dict: typing.Dict[ str, typing.Dict[str["component_name|subsystem_name|size"], str]] = cls.get_elf_info_from_rom_result( rom_result_json) - process_elf_dict: typing.Dict[str, typing.List[str]] = cls.get_process_so_relationship(xml_path, cfg_path, - profile_path) + process_elf_dict: typing.Dict[str, typing.List[str]] = cls.get_process_so_relationship(cfg_path, + json_path) process_size_dict: typing.Dict[str, int] = cls.process_hidumper_info( device_num, ss) result_dict: typing.Dict[str, typing.Dict[str, typing.Any]] = dict() @@ -493,7 +485,7 @@ class RamAnalyzer: # 得到进程相关的elf文件list so_list: list = get(process_name, process_elf_dict) if so_list is None: - print("warning: process '{}' not found in .xml or .cfg".format( + print("warning: process '{}' not found in .json or .cfg".format( process_name)) result_dict[process_name] = dict() result_dict[process_name]["size"] = process_size @@ -571,8 +563,8 @@ def get_args(): ) parser.add_argument("-v", "-version", action="version", version=f"version {VERSION}") - parser.add_argument("-x", "--xml_path", type=str, required=True, - help="path of xml file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile") + parser.add_argument("-s", "--json_path", type=str, required=True, + help="path of sa json file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile") parser.add_argument("-c", "--cfg_path", type=str, required=True, help="path of cfg files. eg: -c ./cfgs/") parser.add_argument("-j", "--rom_result", type=str, default="./rom_analysis_result.json", @@ -591,11 +583,13 @@ def get_args(): args = parser.parse_args() return args +def abspath(path:str)->str: + return os.path.abspath(os.path.expanduser(path)) if __name__ == '__main__': args = get_args() - cfg_path = args.cfg_path - profile_path = args.xml_path + cfg_path = abspath(args.cfg_path) + profile_path = abspath(args.json_path) rom_result = args.rom_result device_num = args.device_num output_filename = args.output_filename -- Gitee From ec72a22377c1f9314c7c1fdfb235d1d92d5d89bf Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Fri, 23 Jun 2023 18:55:28 +0800 Subject: [PATCH 2/3] =?UTF-8?q?[rom=5Fram=5Fanalyzer]rk=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E5=8F=AF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: aodongbiao --- .../lite_small/src/rom_analysis.py | 143 +++++++------ tools/rom_ram_analyzer/standard/README.md | 2 +- .../standard/pkgs/basic_tool.py | 34 +++ .../standard/pkgs/gn_common_tool.py | 80 +++++-- .../rom_ram_analyzer/standard/ram_analyzer.py | 197 ++++++++++-------- .../rom_ram_analyzer/standard/rom_analyzer.py | 117 ++++++++--- 6 files changed, 376 insertions(+), 197 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py index efa1350..5e263f2 100644 --- a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py +++ b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py @@ -88,7 +88,7 @@ class RomAnalysisTool: if (not sub_path) or (os.sep not in sub_path): return # 将其他目录添加到dir_list - t, sub_sub_path = sub_path.split(os.sep, 1) # 如果是c/e,分割成c,e + t, sub_sub_path = sub_path.split(os.sep, 1) # 如果是c/e,分割成c,e t = os.path.join(rela_path, t) if t in dir_list: dir_list.remove(t) @@ -107,7 +107,7 @@ class RomAnalysisTool: logging.error( f"product_name '{product_name}' not found in the config.yaml") exit(1) - product_path_dit: Dict[str, str] = dict() # 存储编译产物的类型及目录 + product_path_dit: Dict[str, str] = dict() # 存储编译产物的类型及目录 root_dir = product_dir.get("root") root_dir = os.path.join(project_path, root_dir) relative_dir: Dict[str, str] = product_dir.get("relative") @@ -120,14 +120,14 @@ class RomAnalysisTool: product_path_dit[k] = os.path.join(root_dir, v) # 查找编译产物信息 # product_dict格式: {"so": ["a.so", "b.so"]} - product_dict: Dict[str, List[str]] = dict() # 存储编译产物的名称 + product_dict: Dict[str, List[str]] = dict() # 存储编译产物的名称 for k, v in product_path_dit.items(): if not os.path.exists(v): logging.warning(f"dir '{v}' not exist") - product_dict[k] = BasicTool.find_files_with_pattern(v) # v是全路径 + product_dict[k] = BasicTool.find_files_with_pattern(v) # v是全路径 if product_dir.get("rest"): rest_dir_list: List[str] = os.listdir( - root_dir) # 除了配置在relative下之外的所有剩余目录,全部归到etc下 + root_dir) # 除了配置在relative下之外的所有剩余目录,全部归到etc下 for v in relative_dir.values(): if v in rest_dir_list: rest_dir_list.remove(v) @@ -152,14 +152,14 @@ class RomAnalysisTool: @classmethod def _put(cls, sub: str, com: str, unit: Dict, rom_size_dict: Dict, com_size_baseline: str = str()): size = unit.get("size") - if not rom_size_dict.get("size"): # 总大小 + if not rom_size_dict.get("size"): # 总大小 rom_size_dict["size"] = 0 - if not rom_size_dict.get(sub): # 子系统大小 + if not rom_size_dict.get(sub): # 子系统大小 rom_size_dict[sub]: Dict[str, Dict] = dict() rom_size_dict[sub]["size"] = 0 rom_size_dict[sub]["count"] = 0 - if not rom_size_dict.get(sub).get(com): # 部件 + if not rom_size_dict.get(sub).get(com): # 部件 rom_size_dict.get(sub)[com] = dict() rom_size_dict[sub][com]["filelist"] = list() rom_size_dict[sub][com]["size"] = 0 @@ -329,7 +329,71 @@ class RomAnalysisTool: del v[index] break - @ classmethod + @classmethod + def _iterate_all_template_type(cls, type_list: List[str], gn_info: Dict, gn_info_file: str, base_name: str, + rom_ram_baseline: Dict, rom_size_dict: Dict, f: str, size: int): + find_flag = False + component_rom_baseline = None + for tn in type_list: # tn example: ohos_shared_library + if find_flag: # 如果已经在前面的template中找到了,后面的就不必再查找 + break + output_dict: Dict[str, Dict] = gn_info.get( + tn) # 这个模板对应的所有可能编译产物 + if not output_dict: + logging.warning( + f"'{tn}' not found in the {gn_info_file}") + continue + d = output_dict.get(base_name) + if not d: + continue + d["size"] = size + d["file_name"] = f.replace(project_path, "") + if rom_ram_baseline.get(d["subsystem_name"]) and rom_ram_baseline.get(d["subsystem_name"]).get(d["component_name"]): + component_rom_baseline = rom_ram_baseline.get( + d["subsystem_name"]).get(d["component_name"]).get("rom") + cls._put(d["subsystem_name"], + d["component_name"], d, rom_size_dict, component_rom_baseline) + find_flag = True + if not find_flag: # 如果指定序列中的template都没有查找到,则模糊匹配 + # fuzzy match + psesudo_gn, sub, com = cls._fuzzy_match(f) + if sub and com: + if rom_ram_baseline.get(sub) and rom_ram_baseline.get(sub).get(com): + component_rom_baseline = rom_ram_baseline.get( + sub).get(com).get("baseline") + cls._put(sub, com, { + "subsystem_name": sub, + "component_name": com, + "psesudo_gn_path": psesudo_gn, + "description": "fuzzy match", + "file_name": f.replace(project_path, ""), + "size": size, + }, rom_size_dict, component_rom_baseline) + find_flag = True + if not find_flag: # 模糊匹配都没有匹配到的,归属到NOTFOUND + cls._put("NOTFOUND", "NOTFOUND", { + "file_name": f.replace(project_path, ""), + "size": size, + }, rom_size_dict) + + @classmethod + def _subsystem_component_for_all_product_file(cls, product_dict: Dict[str, List[str]], query_order: Dict[str, List[str]], + gn_info: Dict, gn_info_file: str, rom_ram_baseline: Dict, rom_size_dict: Dict): + for t, l in product_dict.items(): + for f in l: # 遍历所有文件 + if os.path.isdir(f): + continue + type_list = query_order.get(t) + _, base_name = os.path.split(f) + size = os.path.getsize(f) + if not type_list: + logging.warning( + f"'{t}' not found in query_order of the config.yaml") + break + cls._iterate_all_template_type( + type_list, gn_info, gn_info_file, base_name, rom_ram_baseline, rom_size_dict, f, size) + + @classmethod def analysis(cls, product_name: str, product_dict: Dict[str, List[str]]): """analysis the rom of lite/small product @@ -346,69 +410,18 @@ class RomAnalysisTool: project_path) with open("rom_ram_baseline.json", 'w', encoding='utf-8') as f: json.dump(rom_ram_baseline, f, indent=4) - gn_info_file = configs["gn_info_file"] # filename to save gn_info + gn_info_file = configs["gn_info_file"] # filename to save gn_info with open(gn_info_file, 'r', encoding='utf-8') as f: gn_info = json.load(f) query_order: Dict[str, List[str] - ] = configs[product_name]["query_order"] # query order of the gn template to be matched - query_order["etc"] = configs["target_type"] # etc会查找所有的template + ] = configs[product_name]["query_order"] # query order of the gn template to be matched + query_order["etc"] = configs["target_type"] # etc会查找所有的template rom_size_dict: Dict = dict() if "manual_config" in configs[product_name].keys(): cls._match_manual_configured( configs[product_name]["manual_config"], product_dict, configs[product_name]["product_dir"]["root"], rom_size_dict) - for t, l in product_dict.items(): - for f in l: # 遍历所有文件 - if os.path.isdir(f): - continue - find_flag = False - type_list = query_order.get(t) - _, base_name = os.path.split(f) - size = os.path.getsize(f) - if not type_list: - logging.warning( - f"'{t}' not found in query_order of the config.yaml") - break - for tn in type_list: # tn example: ohos_shared_library - if find_flag: # 如果已经在前面的template中找到了,后面的就不必再查找 - break - output_dict: Dict[str, Dict] = gn_info.get( - tn) # 这个模板对应的所有可能编译产物 - if not output_dict: - logging.warning( - f"'{tn}' not found in the {gn_info_file}") - continue - d = output_dict.get(base_name) - if not d: - continue - d["size"] = size - d["file_name"] = f.replace(project_path, "") - if rom_ram_baseline.get(d["subsystem_name"]) and rom_ram_baseline.get(d["subsystem_name"]).get(d["component_name"]): - component_rom_baseline = rom_ram_baseline.get( - d["subsystem_name"]).get(d["component_name"]).get("rom") - cls._put(d["subsystem_name"], - d["component_name"], d, rom_size_dict, component_rom_baseline) - find_flag = True - if not find_flag: # 如果指定序列中的template都没有查找到,则模糊匹配 - # fuzzy match - psesudo_gn, sub, com = cls._fuzzy_match(f) - if sub and com: - if rom_ram_baseline.get(sub) and rom_ram_baseline.get(sub).get(com): - component_rom_baseline = rom_ram_baseline.get( - sub).get(com).get("baseline") - cls._put(sub, com, { - "subsystem_name": sub, - "component_name": com, - "psesudo_gn_path": psesudo_gn, - "description": "fuzzy match", - "file_name": f.replace(project_path, ""), - "size": size, - }, rom_size_dict, component_rom_baseline) - find_flag = True - if not find_flag: # 模糊匹配都没有匹配到的,归属到NOTFOUND - cls._put("NOTFOUND", "NOTFOUND", { - "file_name": f.replace(project_path, ""), - "size": size, - }, rom_size_dict) + cls._subsystem_component_for_all_product_file( + product_dict, query_order, gn_info, gn_info_file, rom_ram_baseline, rom_size_dict) if unit_adapt: cls._result_unit_adaptive(rom_size_dict) with open(configs[product_name]["output_name"], 'w', encoding='utf-8') as f: diff --git a/tools/rom_ram_analyzer/standard/README.md b/tools/rom_ram_analyzer/standard/README.md index 81008bf..250932d 100644 --- a/tools/rom_ram_analyzer/standard/README.md +++ b/tools/rom_ram_analyzer/standard/README.md @@ -90,7 +90,7 @@ 基于out/{product_name}/packages/phone下所有cfg文件、out/{product_name}/packages/phone/system/profile下所有json文件，rom的分析结果，（rom_ram_baseline.json——可以在rom分析阶段通过-b参数生成）分析各进程及对应部件的ram占用（默认取Pss） -收集cfg、json文件的可供参考命令：`mkdir cfgs && cp $(find ~/oh/out/rk3568/packages/phone -name *.cfs | xargs) cfgs` +收集cfg、json文件的可供参考命令：`mkdir cfgs && cp $(find ~/oh/out/rk3568/packages/phone -name *.cfg | xargs) cfgs` 结果以json与xls格式存储，其中，json格式是必输出的，xls格式需要-e参数控制。 diff --git a/tools/rom_ram_analyzer/standard/pkgs/basic_tool.py b/tools/rom_ram_analyzer/standard/pkgs/basic_tool.py index f5561de..73ae952 100644 --- a/tools/rom_ram_analyzer/standard/pkgs/basic_tool.py +++ b/tools/rom_ram_analyzer/standard/pkgs/basic_tool.py @@ -2,6 +2,7 @@ import sys import typing import os import glob +import re from pathlib import Path from typing import * @@ -17,6 +18,23 @@ def unit_adaptive(size: int) -> str: return str(round(size,2))+unit_list[index] class BasicTool: + @classmethod + def match_paragraph(cls, content: str, start_pattern: str = r"\w+\(\".*?\"\) *{", end_pattern: str = "\}") -> \ + Iterator[re.Match]: + """ + 匹配代码段，支持单行 + 注意：ptrn中已经包含前面的空格，所以start_pattern中可以省略 + :param content: 被匹配的字符串 + :param start_pattern: 模式的开头 + :param end_pattern: 模式的结尾 + :return: 匹配到的段落的迭代器 + """ + ptrn = r'^( *){s}(?#匹配开头).*?(?#中间非贪婪)\1(?#如果开头前面有空格,则结尾的前面应该有相同数量的空格)?{e}$(?#匹配结尾)'.format( + s=start_pattern, e=end_pattern) + ptrn = re.compile(ptrn, re.M | re.S) + result = re.finditer(ptrn, content) + return result + @classmethod def find_all_files(cls, folder: str, real_path: bool = True, apply_abs: bool = True, de_duplicate: bool = True, p_filter: typing.Callable = lambda x: True) -> list: @@ -36,6 +54,22 @@ class BasicTool: def get_abs_path(cls, path: str) -> str: return os.path.abspath(os.path.expanduser(path)) + @classmethod + def re_group_1(cls, content: str, pattern: str, **kwargs) -> str: + """ + 匹配正则表达式，如果有匹配到内容，返回group(1)的内容 + :param content: 要被匹配的内容 + :param pattern: 进行匹配的模式 + :return: 匹配到的结果（group(1)） + TODO 对（）的检查应该更严格 + """ + if not (r'(' in pattern and r')' in pattern): + raise ValueError("parentheses'()' must in the pattern") + result = re.search(pattern, content, **kwargs) + if result: + return result.group(1) + return str() + @classmethod def execute(cls, cmd: str, post_processor: Callable[[Text], Text] = lambda x:x) -> Any: """ diff --git a/tools/rom_ram_analyzer/standard/pkgs/gn_common_tool.py b/tools/rom_ram_analyzer/standard/pkgs/gn_common_tool.py index 11c2718..702d379 100644 --- a/tools/rom_ram_analyzer/standard/pkgs/gn_common_tool.py +++ b/tools/rom_ram_analyzer/standard/pkgs/gn_common_tool.py @@ -1,5 +1,12 @@ import os import json +import re +from typing import * + +if __name__ == '__main__': + from basic_tool import BasicTool +else: + from pkgs.basic_tool import BasicTool class GnCommonTool: @@ -98,21 +105,8 @@ class GnCommonTool: return part_name, subsystem_name @classmethod - def find_part_subsystem(cls, gn_file: str, project_path: str) -> tuple: - """ - 查找gn_file对应的part_name和subsystem - 如果在gn中找不到，就到bundle.json中去找 - """ - part_name = None - subsystem_name = None - part_var_flag = False # 标识这个变量从gn中取出的原始值是不是变量 - subsystem_var_flag = False - var_list = list() - part_name_pattern = r"part_name *=\s*\S*" - subsystem_pattern = r"subsystem_name *=\s*\S*" - meta_grep_pattern = "grep -E '{}' {} | head -n 1" - part_cmd = meta_grep_pattern.format(part_name_pattern, gn_file) - subsystem_cmd = meta_grep_pattern.format(subsystem_pattern, gn_file) + def _parse_part_subsystem(cls, part_var_flag: bool, subsystem_var_flag: bool, var_list: List[str], part_cmd: str, subsystem_cmd: str, gn_file: str, project_path: str) -> Tuple[str, str]: + part_name = subsystem_name = None part = os.popen(part_cmd).read().strip() if len(part) != 0: part = part.split('=')[-1].strip() @@ -145,13 +139,63 @@ class GnCommonTool: tuple(var_list), gn_file, project_path)[0] subsystem_name = t if t is not None and len( t) != 0 else subsystem_name - if part_name is not None and subsystem_name is not None: + return part_name, subsystem_name + + @classmethod + def find_part_subsystem(cls, gn_file: str, project_path: str) -> tuple: + """ + 查找gn_file对应的part_name和subsystem + 如果在gn中找不到，就到bundle.json中去找 + """ + part_var_flag = False # 标识这个变量从gn中取出的原始值是不是变量 + subsystem_var_flag = False + var_list = list() + part_name_pattern = r"part_name *=\s*\S*" + subsystem_pattern = r"subsystem_name *=\s*\S*" + meta_grep_pattern = "grep -E '{}' {} | head -n 1" + part_cmd = meta_grep_pattern.format(part_name_pattern, gn_file) + subsystem_cmd = meta_grep_pattern.format(subsystem_pattern, gn_file) + + part_name, subsystem_name = cls._parse_part_subsystem(part_var_flag, subsystem_var_flag, + var_list, part_cmd, subsystem_cmd, gn_file, project_path) + if part_name and subsystem_name: return part_name, subsystem_name # 如果有一个没有找到，就要一层层去找bundle.json文件 t_part_name, t_subsystem_name = cls.__find_part_subsystem_from_bundle( gn_file, stop_tail=project_path) - if t_part_name is not None: + if t_part_name: part_name = t_part_name - if t_subsystem_name is not None: + if t_subsystem_name: subsystem_name = t_subsystem_name return part_name, subsystem_name + +class GnVariableParser: + @classmethod + def string_parser(cls, var: str, content: str) -> str: + """ + 解析值为字符串的变量,没有对引号进行去除,如果是a = b这种b为变量的,则无法匹配 + :param content: 要进行解析的内容 + :param var: 变量名 + :return: 变量值[str] + """ + result = BasicTool.re_group_1( + content, r"{} *= *[\n]?(\".*?\")".format(var), flags=re.S | re.M) + return result + + @classmethod + def list_parser(cls, var: str, content: str) -> List[str]: + """ + 解析值为列表的变量，list的元素必须全为数字或字符串,且没有对引号进行去除,如果是a = b这种b为变量的,则无法匹配 + :param var: 变量名 + :param content: 要进行 + :return: 变量值[List] + """ + result = BasicTool.re_group_1( + content, r"{} *= *(\[.*?\])".format(var), flags=re.S | re.M) + result_list = list() + for item in result.lstrip('[').rstrip(']').split('\n'): + item = item.strip().strip(',"') + if not item: + continue + result_list.append(item) + return result_list diff --git a/tools/rom_ram_analyzer/standard/ram_analyzer.py b/tools/rom_ram_analyzer/standard/ram_analyzer.py index c6b9fd2..3e1a399 100644 --- a/tools/rom_ram_analyzer/standard/ram_analyzer.py +++ b/tools/rom_ram_analyzer/standard/ram_analyzer.py @@ -139,9 +139,9 @@ class RamAnalyzer: # 如果第一列不是数字（pid），就过 if not processed or not processed[0].isnumeric(): continue - name = processed[1] # 否则的话就取名字，和对应的size + name = processed[1] # 否则的话就取名字，和对应的size size = int(processed[cls.__ss_dict.get(ss)]) * \ - 1024 # kilo byte to byte + 1024 # kilo byte to byte full_process_name = find_full_process_name(name) if not full_process_name: print( @@ -181,20 +181,20 @@ class RamAnalyzer: print("warning: {} not exist or not a json file".format(file_path)) return with open(file_path, 'r', encoding='utf-8') as f: - j_content:typing.Dict[str, typing.Any] = json.load(f) + j_content: typing.Dict[str, typing.Any] = json.load(f) if "process" not in j_content.keys() or "systemability" not in j_content.keys(): - print(f"warning: {file_path} has no field 'process' or 'systemability'") + print( + f"warning: {file_path} has no field 'process' or 'systemability'") return - process_name:str = j_content.get("process") - elf_list:typing.List[str] = list() + process_name: str = j_content.get("process") + elf_list: typing.List[str] = list() for sa in j_content.get("systemability"): - libpath:str = sa.get("libpath") + libpath: str = sa.get("libpath") if not libpath: continue elf_list.append(libpath) result_dict[process_name] = elf_list - @classmethod def get_elf_info_from_rom_result(cls, rom_result_json: str) -> typing.Dict[str, typing.Dict[str, str]]: """ @@ -249,7 +249,7 @@ class RamAnalyzer: result_dict.get(process_name).append(os.path.split(first)[-1]) @classmethod - def get_process_so_relationship(cls,cfg_path: str, profile_path: str) -> typing.Dict[ + def get_process_so_relationship(cls, cfg_path: str, profile_path: str) -> typing.Dict[ str, typing.List[str]]: """ parse the relationship between process and elf file @@ -270,7 +270,7 @@ class RamAnalyzer: return process_elf_dict @classmethod - def __save_result_as_excel(cls, data_dict: dict, filename: str, ss: str, baseline_file: str, unit_adapt:bool): + def __save_result_as_excel(cls, data_dict: dict, filename: str, ss: str, baseline_file: str, unit_adapt: bool): """ 保存结果到excel中 子系统：{ @@ -403,34 +403,114 @@ class RamAnalyzer: continue component_info["baseline"] = baseline_dict[subsystem_name][component_name].get( "ram") + @classmethod - def refactored_result_unit_adaptive(cls, result_dict:Dict[str,Dict])->None: - for subsystem_name, subsystem_info in result_dict.items(): + def refactored_result_unit_adaptive(cls, result_dict: Dict[str, Dict]) -> None: + for _, subsystem_info in result_dict.items(): sub_size = unit_adaptive(subsystem_info["size"]) del subsystem_info["size"] - for component_name, component_info in subsystem_info.items(): + for _, component_info in subsystem_info.items(): com_size = unit_adaptive(component_info["size"]) del component_info["size"] - for process_name, process_info in component_info.items(): + for _, process_info in component_info.items(): pro_size = unit_adaptive(process_info["size"]) del process_info["size"] for elf_name, elf_size in process_info["elf"].items(): - process_info["elf"][elf_name]=unit_adaptive(elf_size) + process_info["elf"][elf_name] = unit_adaptive(elf_size) process_info["size"] = pro_size component_info["size"] = com_size subsystem_info["size"] = sub_size - + @classmethod - def analysis(cls, cfg_path: str, json_path: str, rom_result_json: str, device_num: str, - output_file: str, ss: str, output_excel: bool, baseline_file: str, unit_adapt:bool): - """ - process size subsystem/component so so_size - """ + def verify(cls, device_num: str) -> bool: if not HDCTool.verify_hdc(): print("error: Command 'hdc' not found") - return + return False if not HDCTool.verify_device(device_num): print("error: {} is inaccessible or not found".format(device_num)) + return False + return True + + @classmethod + def get(key: typing.Any, dt: typing.Dict[str, typing.Any]): + for k, v in dt.items(): + if k.startswith(key) or (len(v) > 0 and key == v[0]): + # 要么uinput_inject的对应key为mmi_uinput_inject。对于此类特殊处理，即：如果service_name找不到，但是直接执行的bin等于这个名字，也认为找到 + return v + + @classmethod + def _process_process_info(cls, process_name: str, result_dict: Dict, process_size: int, + rom_result_dict: Dict, process_elf_dict: Dict, so_info_dict: Dict,): + # 如果部件是init,特殊处理 + if process_name == "init": + _, elf, _, _, size = cls.find_elf_size_from_rom_result(process_name, "startup", "init", + lambda x, y: os.path.split(y)[ + -1].lower() == x.lower(), + rom_result_dict) + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + result_dict[process_name]["startup"] = dict() + result_dict[process_name]["startup"]["init"] = dict() + result_dict[process_name]["startup"]["init"][elf if len( + elf) != 0 else "UNKNOWN"] = size + return + # 如果是hap，特殊处理 + if (process_name.startswith("com.") or process_name.startswith("ohos.")): + _, hap_name, subsystem_name, component_name, size = cls.find_elf_size_from_rom_result(process_name, "*", "*", + lambda x, y: len( + y.split( + '/')) >= 3 and x.lower().startswith( + y.split('/')[2].lower()), + rom_result_dict) + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + result_dict[process_name][subsystem_name] = dict() + result_dict[process_name][subsystem_name][component_name] = dict() + result_dict[process_name][subsystem_name][component_name][hap_name if len( + hap_name) != 0 else "UNKNOWN"] = size + return + # 得到进程相关的elf文件list + so_list: list = cls.get(process_name, process_elf_dict) + if so_list is None: + print("warning: process '{}' not found in .json or .cfg".format( + process_name)) + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + result_dict[process_name]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"]["UNKNOWN"] = int() + return + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + for so in so_list: + unit = so_info_dict.get(so) + if unit is None: + result_dict[process_name]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"][so] = int() + print("warning: '{}' in {} not found in json from rom analysis result".format( + so, process_name)) + continue + component_name = unit.get("component_name") + subsystem_name = unit.get("subsystem_name") + so_size = unit.get("size") + if result_dict.get(process_name).get(subsystem_name) is None: + result_dict[process_name][subsystem_name] = dict() + if result_dict.get(process_name).get(subsystem_name).get(component_name) is None: + result_dict[process_name][subsystem_name][component_name] = dict() + result_dict[process_name][subsystem_name][component_name][so] = so_size + + @classmethod + def _collect_sa_profile_info(cls, project_path:str): + ... + + @classmethod + def analysis(cls, cfg_path: str, json_path: str, rom_result_json: str, device_num: str, + output_file: str, ss: str, output_excel: bool, baseline_file: str, unit_adapt: bool): + """ + process size subsystem/component so so_size + """ + if not cls.verify(device_num): return with open(rom_result_json, 'r', encoding='utf-8') as f: rom_result_dict: typing.Dict = json.loads(f.read()) @@ -443,75 +523,12 @@ class RamAnalyzer: process_size_dict: typing.Dict[str, int] = cls.process_hidumper_info( device_num, ss) result_dict: typing.Dict[str, typing.Dict[str, typing.Any]] = dict() - - def get(key: typing.Any, dt: typing.Dict[str, typing.Any]): - for k, v in dt.items(): - if k.startswith(key) or (len(v) > 0 and key == v[0]): - # 要么uinput_inject的对应key为mmi_uinput_inject。对于此类特殊处理，即：如果service_name找不到，但是直接执行的bin等于这个名字，也认为找到 - return v - - for process_name, process_size in process_size_dict.items(): # 从进程出发 + for process_name, process_size in process_size_dict.items(): # 从进程出发 if not process_name: print("warning: an empty 'process_name' has been found.") continue - # 如果部件是init,特殊处理 - if process_name == "init": - _, elf, _, _, size = cls.find_elf_size_from_rom_result(process_name, "startup", "init", - lambda x, y: os.path.split(y)[ - -1].lower() == x.lower(), - rom_result_dict) - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - result_dict[process_name]["startup"] = dict() - result_dict[process_name]["startup"]["init"] = dict() - result_dict[process_name]["startup"]["init"][elf if len( - elf) != 0 else "UNKNOWN"] = size - continue - # 如果是hap，特殊处理 - if (process_name.startswith("com.") or process_name.startswith("ohos.")): - _, hap_name, subsystem_name, component_name, size = cls.find_elf_size_from_rom_result(process_name, "*", "*", - lambda x, y: len( - y.split( - '/')) >= 3 and x.lower().startswith( - y.split('/')[2].lower()), - rom_result_dict) - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - result_dict[process_name][subsystem_name] = dict() - result_dict[process_name][subsystem_name][component_name] = dict() - result_dict[process_name][subsystem_name][component_name][hap_name if len( - hap_name) != 0 else "UNKNOWN"] = size - continue - # 得到进程相关的elf文件list - so_list: list = get(process_name, process_elf_dict) - if so_list is None: - print("warning: process '{}' not found in .json or .cfg".format( - process_name)) - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - result_dict[process_name]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"]["UNKNOWN"] = int() - continue - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - for so in so_list: - unit = so_info_dict.get(so) - if unit is None: - result_dict[process_name]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"][so] = int() - print("warning: '{}' in {} not found in json from rom analysis result".format( - so, process_name)) - continue - component_name = unit.get("component_name") - subsystem_name = unit.get("subsystem_name") - so_size = unit.get("size") - if result_dict.get(process_name).get(subsystem_name) is None: - result_dict[process_name][subsystem_name] = dict() - if result_dict.get(process_name).get(subsystem_name).get(component_name) is None: - result_dict[process_name][subsystem_name][component_name] = dict() - result_dict[process_name][subsystem_name][component_name][so] = so_size + cls._process_process_info( + process_name, result_dict, process_size, rom_result_dict, process_elf_dict, so_info_dict) base_dir, _ = os.path.split(output_file) if len(base_dir) != 0 and not os.path.isdir(base_dir): os.makedirs(base_dir, exist_ok=True) @@ -583,9 +600,11 @@ def get_args(): args = parser.parse_args() return args -def abspath(path:str)->str: + +def abspath(path: str) -> str: return os.path.abspath(os.path.expanduser(path)) + if __name__ == '__main__': args = get_args() cfg_path = abspath(args.cfg_path) diff --git a/tools/rom_ram_analyzer/standard/rom_analyzer.py b/tools/rom_ram_analyzer/standard/rom_analyzer.py index 2536a6c..8619f93 100644 --- a/tools/rom_ram_analyzer/standard/rom_analyzer.py +++ b/tools/rom_ram_analyzer/standard/rom_analyzer.py @@ -22,10 +22,11 @@ import sys import typing from copy import deepcopy from typing import * - +import re +import subprocess from pkgs.rom_ram_baseline_collector import RomRamBaselineCollector from pkgs.basic_tool import BasicTool, unit_adaptive -from pkgs.gn_common_tool import GnCommonTool +from pkgs.gn_common_tool import GnCommonTool, GnVariableParser from pkgs.simple_excel_writer import SimpleExcelWriter debug = bool(sys.gettrace()) @@ -33,27 +34,80 @@ debug = bool(sys.gettrace()) NOTFOUND = "NOTFOUND" +class PreCollector: + """ + collect some info that system_module_info.json dosn't contains + """ + + def __init__(self, project_path: str) -> None: + self.info_dict: Dict[str, Any] = dict() + self.project_path = BasicTool.get_abs_path(project_path) + self.result_dict = dict() + + def _process_single_sa(self, item: str, start_pattern: str): + gn, _, _ = item.split(':') + with open(gn, 'r', encoding='utf-8') as f: + content = f.read() + p_itr: Iterator[re.Match] = BasicTool.match_paragraph( + content=content, start_pattern=start_pattern) + for p in p_itr: + p_content = p.group() + files: List[str] = GnVariableParser.list_parser( + "sources", p_content) + component_name, subsystem_name = GnCommonTool.find_part_subsystem( + gn, self.project_path) + for f in files: + f = f.split('/')[-1] + self.result_dict[f] = { + "subsystem_name": subsystem_name, + "component_name": component_name, + "gn_path": gn + } + + def collect_sa_profile(self): + grep_kw = r"ohos_sa_profile" + grep_cmd = f"grep -rn '{grep_kw}' --include=BUILD.gn {self.project_path}" + content = BasicTool.execute( + grep_cmd, post_processor=lambda x: x.split('\n')) + for item in content: + if not item: + continue + self._process_single_sa(item, start_pattern=grep_kw) + + class RomAnalyzer: @classmethod def __collect_product_info(cls, system_module_info_json: Text, - project_path: Text) -> Dict[Text, Dict[Text, Text]]: + project_path: Text, extra_info: Dict[str, Dict]) -> Dict[Text, Dict[Text, Text]]: """ 根据system_module_info.json生成target字典 + format: + { + "{file_name}":{ + "{subsytem_name}": abc, + "{component_name}": def, + "{gn_path}": ghi + } + } + if the unit of system_module_info.json has not field "label" and the "type" is "sa_profile", + find the subsystem_name and component_name in the BUILD.gn """ with open(system_module_info_json, 'r', encoding='utf-8') as f: product_list = json.loads(f.read()) project_path = BasicTool.get_abs_path(project_path) product_info_dict: Dict[Text, Dict[Text, Text]] = dict() for unit in product_list: + cs_flag = False dest: List = unit.get("dest") - if dest is None: + if not dest: print("warning: keyword 'dest' not found in {}".format( system_module_info_json)) continue label: Text = unit.get("label") gn_path = component_name = subsystem_name = None if label: + cs_flag = True gn_path = os.path.join(project_path, label.split(':')[ 0].lstrip('/'), "BUILD.gn") component_name = unit.get("part_name") @@ -66,11 +120,18 @@ class RomAnalyzer: else: print("warning: keyword 'label' not found in {}".format(unit)) for target in dest: - product_info_dict[target] = { - "component_name": component_name, - "subsystem_name": subsystem_name, - "gn_path": gn_path, - } + if cs_flag: + product_info_dict[target] = { + "component_name": component_name, + "subsystem_name": subsystem_name, + "gn_path": gn_path, + } + else: + tmp = target.split('/')[-1] + pre_info = extra_info.get(tmp) + if not pre_info: + continue + product_info_dict[target] = pre_info return product_info_dict @classmethod @@ -142,12 +203,12 @@ class RomAnalyzer: return baseline_dict.get(subsystem_name).get(component_name).get("rom") size = unit.get("size") relative_filepath = unit.get("relative_filepath") - if result_dict.get(subsystem_name) is None: # 子系统 + if result_dict.get(subsystem_name) is None: # 子系统 result_dict[subsystem_name] = dict() result_dict[subsystem_name]["size"] = 0 result_dict[subsystem_name]["file_count"] = 0 - if result_dict.get(subsystem_name).get(component_name) is None: # 部件 + if result_dict.get(subsystem_name).get(component_name) is None: # 部件 result_dict[subsystem_name][component_name] = dict() result_dict[subsystem_name][component_name]["size"] = 0 result_dict[subsystem_name][component_name]["file_count"] = 0 @@ -162,18 +223,18 @@ class RomAnalyzer: result_dict[subsystem_name][component_name][relative_filepath] = size @classmethod - def result_unit_adaptive(self, result_dict:Dict[str,Dict])->None: + def result_unit_adaptive(self, result_dict: Dict[str, Dict]) -> None: for subsystem_name, subsystem_info in result_dict.items(): - size = unit_adaptive(subsystem_info["size"]) - count = subsystem_info["file_count"] - if "size" in subsystem_info.keys(): - del subsystem_info["size"] - if "file_count" in subsystem_info.keys(): - del subsystem_info["file_count"] - for component_name, component_info in subsystem_info.items(): - component_info["size"] = unit_adaptive(component_info["size"]) - subsystem_info["size"] = size - subsystem_info["file_count"] = count + size = unit_adaptive(subsystem_info["size"]) + count = subsystem_info["file_count"] + if "size" in subsystem_info.keys(): + del subsystem_info["size"] + if "file_count" in subsystem_info.keys(): + del subsystem_info["file_count"] + for component_name, component_info in subsystem_info.items(): + component_info["size"] = unit_adaptive(component_info["size"]) + subsystem_info["size"] = size + subsystem_info["file_count"] = count @classmethod def analysis(cls, system_module_info_json: Text, product_dirs: List[str], @@ -193,8 +254,11 @@ class RomAnalyzer: phone_dir = os.path.join( project_path, "out", product_name, "packages", "phone") product_dirs = [os.path.join(phone_dir, d) for d in product_dirs] + pre_collector = PreCollector(project_path) + pre_collector.collect_sa_profile() + extra_product_info_dict: Dict[str, Dict] = pre_collector.result_dict product_info_dict = cls.__collect_product_info( - system_module_info_json, project_path) # 所有产物信息 + system_module_info_json, project_path, extra_info=extra_product_info_dict) # collect product info from json file result_dict: Dict[Text:Dict] = dict() for d in product_dirs: file_list: List[Text] = BasicTool.find_all_files(d) @@ -203,7 +267,10 @@ class RomAnalyzer: relative_filepath = f.replace(phone_dir, "").lstrip(os.sep) unit: Dict[Text, Any] = product_info_dict.get( relative_filepath) - if unit is None: + if not unit: + bf = f.split('/')[-1] + unit: Dict[Text, Any] = product_info_dict.get(bf) + if not unit: unit = dict() unit["size"] = size unit["relative_filepath"] = relative_filepath @@ -258,3 +325,5 @@ if __name__ == '__main__': unit_adapt = args.unit_adaptive RomAnalyzer.analysis(module_info_json, product_dirs, project_path, product_name, output_file, output_excel, add_baseline, unit_adapt) + # collector = PreCollector("~/oh_nomodify") + # collector.collect_sa_profile() -- Gitee From 26b6655b8e404a253c1004c5514a6b185c81a2d4 Mon Sep 17 00:00:00 2001 From: aodongbiao Date: Fri, 23 Jun 2023 19:12:16 +0800 Subject: [PATCH 3/3] [rom_ram_analyzer]verified lite/small' Signed-off-by: aodongbiao --- tools/rom_ram_analyzer/lite_small/README.md | 2 +- .../lite_small/src/rom_analysis.py | 3 +- tools/rom_ram_analyzer/standard/README.md | 9 +- .../rom_ram_analyzer/standard/ram_analyzer.py | 164 ++++++++---------- .../rom_ram_analyzer/standard/rom_analyzer.py | 4 +- 5 files changed, 84 insertions(+), 98 deletions(-) diff --git a/tools/rom_ram_analyzer/lite_small/README.md b/tools/rom_ram_analyzer/lite_small/README.md index e9e66a6..468a512 100644 --- a/tools/rom_ram_analyzer/lite_small/README.md +++ b/tools/rom_ram_analyzer/lite_small/README.md @@ -15,7 +15,7 @@ ## 代码思路 -1. 扫描BUILD.gn文件,收集各个target的编译产物及其对应的component_name, subsystem_name信息,并存储到config.yaml中的gn_info_file字段指定的json文件中 +1. 扫描BUILD.gn文件,收集各个target的编译产物及其对应的component_name, subsystem_name信息,并存储到config.yaml中的gn_info_file字段指定的json文件中。如果BUILD.gn中没有查找到，则直接使用get_subsytem_component.py中预先收集好的数据（根据bundle.json） 2. 根据配置文件config.yaml扫描产品的编译产物目录,得到真实的编译产物信息(主要是大小) 3. 用真实的编译产物与从BUILD.gn中收集的信息进行匹配,从而得到编译产物-大小-所属部件的对应信息 4. 如果匹配失败,会直接利用grep到项目路径下进行模糊搜索,取出现次数top1的BUILD.gn,并根据该BUILD.gn文件去查找子系统和部件 diff --git a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py index 5e263f2..54e3986 100644 --- a/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py +++ b/tools/rom_ram_analyzer/lite_small/src/rom_analysis.py @@ -330,8 +330,7 @@ class RomAnalysisTool: break @classmethod - def _iterate_all_template_type(cls, type_list: List[str], gn_info: Dict, gn_info_file: str, base_name: str, - rom_ram_baseline: Dict, rom_size_dict: Dict, f: str, size: int): + def _iterate_all_template_type(cls, type_list: List[str], gn_info: Dict, gn_info_file: str, base_name: str, rom_ram_baseline: Dict, rom_size_dict: Dict, f: str, size: int): find_flag = False component_rom_baseline = None for tn in type_list: # tn example: ohos_shared_library diff --git a/tools/rom_ram_analyzer/standard/README.md b/tools/rom_ram_analyzer/standard/README.md index 250932d..cef2cbf 100644 --- a/tools/rom_ram_analyzer/standard/README.md +++ b/tools/rom_ram_analyzer/standard/README.md @@ -12,6 +12,10 @@ 主要是rk3568系列,已测试产品包括rk3568、rk3568_mini_system +## 实现思路 + +利用编译构建自动生成的out/rk3568/packages/phone/system_module_info.json中已有的信息重新组织，对于其中没有子系统和部件的文件，手动查找。目前已知ohos_sa_profile没有 + ## 使用说明 前置条件： @@ -30,7 +34,7 @@ 1. `-h`或`--help`命令查看帮助 ```shell > python3 rom_analyzer.py -h - usage: rom_analyzer.py [-h] [-v] -p PROJECT_PATH -j MODULE_INFO_JSON -n PRODUCT_NAME -d PRODUCT_DIR [-b] [-o OUTPUT_FILE] [-e EXCEL] + usage: rom_analyzer.py [-h] [-v] -p PROJECT_PATH -j MODULE_INFO_JSON -n PRODUCT_NAME -d PRODUCT_DIR [-b] [-o OUTPUT_FILE] [-u] [-e EXCEL] analyze rom size of component. @@ -48,6 +52,7 @@ -b, --baseline add baseline of component to the result(-b) or not. -o OUTPUT_FILE, --output_file OUTPUT_FILE basename of output file, default: rom_analysis_result. eg: demo/rom_analysis_result + -u, --unit_adaptive unit adaptive -e EXCEL, --excel EXCEL if output result as excel, default: False. eg: -e True ``` @@ -55,7 +60,7 @@ ```shell python3 rom_analyzer.py -p ~/oh/ -j ~/oh/out/rk3568/packages/phone/system_module_info.json -n rk3568 -d system -d vendor -d updater -e True -b # oh：rootpath of oh - # -b: add baseline of to the result + # -b: add baseline info to the result # -e True：output result in excel format additionally ``` diff --git a/tools/rom_ram_analyzer/standard/ram_analyzer.py b/tools/rom_ram_analyzer/standard/ram_analyzer.py index 3e1a399..e17b065 100644 --- a/tools/rom_ram_analyzer/standard/ram_analyzer.py +++ b/tools/rom_ram_analyzer/standard/ram_analyzer.py @@ -43,7 +43,7 @@ class HDCTool: True：可用 False：不可用 """ - cp = subprocess.run(["hdc"], capture_output=True) + cp = subprocess.run(["hdc", "--help"], capture_output=True) stdout = str(cp.stdout) stderr = str(cp.stderr) return verify_str in stdout or verify_str in stderr @@ -406,13 +406,13 @@ class RamAnalyzer: @classmethod def refactored_result_unit_adaptive(cls, result_dict: Dict[str, Dict]) -> None: - for _, subsystem_info in result_dict.items(): + for subsystem_name, subsystem_info in result_dict.items(): sub_size = unit_adaptive(subsystem_info["size"]) del subsystem_info["size"] - for _, component_info in subsystem_info.items(): + for component_name, component_info in subsystem_info.items(): com_size = unit_adaptive(component_info["size"]) del component_info["size"] - for _, process_info in component_info.items(): + for process_name, process_info in component_info.items(): pro_size = unit_adaptive(process_info["size"]) del process_info["size"] for elf_name, elf_size in process_info["elf"].items(): @@ -421,96 +421,17 @@ class RamAnalyzer: component_info["size"] = com_size subsystem_info["size"] = sub_size - @classmethod - def verify(cls, device_num: str) -> bool: - if not HDCTool.verify_hdc(): - print("error: Command 'hdc' not found") - return False - if not HDCTool.verify_device(device_num): - print("error: {} is inaccessible or not found".format(device_num)) - return False - return True - - @classmethod - def get(key: typing.Any, dt: typing.Dict[str, typing.Any]): - for k, v in dt.items(): - if k.startswith(key) or (len(v) > 0 and key == v[0]): - # 要么uinput_inject的对应key为mmi_uinput_inject。对于此类特殊处理，即：如果service_name找不到，但是直接执行的bin等于这个名字，也认为找到 - return v - - @classmethod - def _process_process_info(cls, process_name: str, result_dict: Dict, process_size: int, - rom_result_dict: Dict, process_elf_dict: Dict, so_info_dict: Dict,): - # 如果部件是init,特殊处理 - if process_name == "init": - _, elf, _, _, size = cls.find_elf_size_from_rom_result(process_name, "startup", "init", - lambda x, y: os.path.split(y)[ - -1].lower() == x.lower(), - rom_result_dict) - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - result_dict[process_name]["startup"] = dict() - result_dict[process_name]["startup"]["init"] = dict() - result_dict[process_name]["startup"]["init"][elf if len( - elf) != 0 else "UNKNOWN"] = size - return - # 如果是hap，特殊处理 - if (process_name.startswith("com.") or process_name.startswith("ohos.")): - _, hap_name, subsystem_name, component_name, size = cls.find_elf_size_from_rom_result(process_name, "*", "*", - lambda x, y: len( - y.split( - '/')) >= 3 and x.lower().startswith( - y.split('/')[2].lower()), - rom_result_dict) - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - result_dict[process_name][subsystem_name] = dict() - result_dict[process_name][subsystem_name][component_name] = dict() - result_dict[process_name][subsystem_name][component_name][hap_name if len( - hap_name) != 0 else "UNKNOWN"] = size - return - # 得到进程相关的elf文件list - so_list: list = cls.get(process_name, process_elf_dict) - if so_list is None: - print("warning: process '{}' not found in .json or .cfg".format( - process_name)) - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - result_dict[process_name]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"]["UNKNOWN"] = int() - return - result_dict[process_name] = dict() - result_dict[process_name]["size"] = process_size - for so in so_list: - unit = so_info_dict.get(so) - if unit is None: - result_dict[process_name]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() - result_dict[process_name]["UNKNOWN"]["UNKNOWN"][so] = int() - print("warning: '{}' in {} not found in json from rom analysis result".format( - so, process_name)) - continue - component_name = unit.get("component_name") - subsystem_name = unit.get("subsystem_name") - so_size = unit.get("size") - if result_dict.get(process_name).get(subsystem_name) is None: - result_dict[process_name][subsystem_name] = dict() - if result_dict.get(process_name).get(subsystem_name).get(component_name) is None: - result_dict[process_name][subsystem_name][component_name] = dict() - result_dict[process_name][subsystem_name][component_name][so] = so_size - - @classmethod - def _collect_sa_profile_info(cls, project_path:str): - ... - @classmethod def analysis(cls, cfg_path: str, json_path: str, rom_result_json: str, device_num: str, output_file: str, ss: str, output_excel: bool, baseline_file: str, unit_adapt: bool): """ process size subsystem/component so so_size """ - if not cls.verify(device_num): + if not HDCTool.verify_hdc(): + print("error: Command 'hdc' not found") + return + if not HDCTool.verify_device(device_num): + print("error: {} is inaccessible or not found".format(device_num)) return with open(rom_result_json, 'r', encoding='utf-8') as f: rom_result_dict: typing.Dict = json.loads(f.read()) @@ -523,12 +444,75 @@ class RamAnalyzer: process_size_dict: typing.Dict[str, int] = cls.process_hidumper_info( device_num, ss) result_dict: typing.Dict[str, typing.Dict[str, typing.Any]] = dict() + + def get(key: typing.Any, dt: typing.Dict[str, typing.Any]): + for k, v in dt.items(): + if k.startswith(key) or (len(v) > 0 and key == v[0]): + # 要么uinput_inject的对应key为mmi_uinput_inject。对于此类特殊处理，即：如果service_name找不到，但是直接执行的bin等于这个名字，也认为找到 + return v + for process_name, process_size in process_size_dict.items(): # 从进程出发 if not process_name: print("warning: an empty 'process_name' has been found.") continue - cls._process_process_info( - process_name, result_dict, process_size, rom_result_dict, process_elf_dict, so_info_dict) + # 如果部件是init,特殊处理 + if process_name == "init": + _, elf, _, _, size = cls.find_elf_size_from_rom_result(process_name, "startup", "init", + lambda x, y: os.path.split(y)[ + -1].lower() == x.lower(), + rom_result_dict) + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + result_dict[process_name]["startup"] = dict() + result_dict[process_name]["startup"]["init"] = dict() + result_dict[process_name]["startup"]["init"][elf if len( + elf) != 0 else "UNKNOWN"] = size + continue + # 如果是hap，特殊处理 + if (process_name.startswith("com.") or process_name.startswith("ohos.")): + _, hap_name, subsystem_name, component_name, size = cls.find_elf_size_from_rom_result(process_name, "*", "*", + lambda x, y: len( + y.split( + '/')) >= 3 and x.lower().startswith( + y.split('/')[2].lower()), + rom_result_dict) + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + result_dict[process_name][subsystem_name] = dict() + result_dict[process_name][subsystem_name][component_name] = dict() + result_dict[process_name][subsystem_name][component_name][hap_name if len( + hap_name) != 0 else "UNKNOWN"] = size + continue + # 得到进程相关的elf文件list + so_list: list = get(process_name, process_elf_dict) + if so_list is None: + print("warning: process '{}' not found in .json or .cfg".format( + process_name)) + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + result_dict[process_name]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"]["UNKNOWN"] = int() + continue + result_dict[process_name] = dict() + result_dict[process_name]["size"] = process_size + for so in so_list: + unit = so_info_dict.get(so) + if unit is None: + result_dict[process_name]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"] = dict() + result_dict[process_name]["UNKNOWN"]["UNKNOWN"][so] = int() + print("warning: '{}' in {} not found in json from rom analysis result".format( + so, process_name)) + continue + component_name = unit.get("component_name") + subsystem_name = unit.get("subsystem_name") + so_size = unit.get("size") + if result_dict.get(process_name).get(subsystem_name) is None: + result_dict[process_name][subsystem_name] = dict() + if result_dict.get(process_name).get(subsystem_name).get(component_name) is None: + result_dict[process_name][subsystem_name][component_name] = dict() + result_dict[process_name][subsystem_name][component_name][so] = so_size base_dir, _ = os.path.split(output_file) if len(base_dir) != 0 and not os.path.isdir(base_dir): os.makedirs(base_dir, exist_ok=True) diff --git a/tools/rom_ram_analyzer/standard/rom_analyzer.py b/tools/rom_ram_analyzer/standard/rom_analyzer.py index 8619f93..87b08c4 100644 --- a/tools/rom_ram_analyzer/standard/rom_analyzer.py +++ b/tools/rom_ram_analyzer/standard/rom_analyzer.py @@ -324,6 +324,4 @@ if __name__ == '__main__': add_baseline = args.baseline unit_adapt = args.unit_adaptive RomAnalyzer.analysis(module_info_json, product_dirs, - project_path, product_name, output_file, output_excel, add_baseline, unit_adapt) - # collector = PreCollector("~/oh_nomodify") - # collector.collect_sa_profile() + project_path, product_name, output_file, output_excel, add_baseline, unit_adapt) \ No newline at end of file -- Gitee