From 42da2f92cb41610293c79f56c90d364571af83c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Wed, 16 Aug 2023 08:29:12 +0000 Subject: [PATCH 01/10] update profiler/merge_profiling_timeline/main.py. change error log and custom merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/profiler/merge_profiling_timeline/main.py b/profiler/merge_profiling_timeline/main.py index eac48f5683..7e782bff3d 100644 --- a/profiler/merge_profiling_timeline/main.py +++ b/profiler/merge_profiling_timeline/main.py @@ -106,8 +106,8 @@ def get_absolute_ts_start_info(pro_path) -> float: if start_json: with open(start_json, "r+") as f: info = json.load(f) - ts_us = float(info.get("collectionTimeBegin"), 0) - ts_ns = float(info.get("clockMonotonicRaw"), 0) + ts_us = float(info.get("collectionTimeBegin")) + ts_ns = float(info.get("clockMonotonicRaw")) if not ts_us and not ts_ns: return 0 return ts_us-ts_ns/1000 @@ -183,15 +183,15 @@ def merge_timeline_events(timeline_file_dict, process_list): if event.get("name") == "process_name" and event.get("ph") == "M": if event.get("args"): proc_pid_dict[event["args"].get("name")] = event.get("pid") - process_list = process_list if process_list else list(proc_pid_dict.keys()) + process_list_tmp = process_list if process_list else list(proc_pid_dict.keys()) # 提取待合并的items的pid merged_pids = set() - for pro in process_list: + for pro in process_list_tmp: pro = " ".join(pro.split("_")) if "_" in pro else pro if pro not in proc_pid_dict.keys(): - print(f"{pro} is invalid item, valid items: {list(proc_pid_dict.keys())}") - continue + print(f"main.py: error argument --items: invalid choice: '{pro}' (choose from {list(proc_pid_dict.keys())})") + return merged_pids.add(proc_pid_dict.get(pro)) for event in cur_events: -- Gitee From 32e37983594e6d0f09762e843233fd202b74634b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Wed, 16 Aug 2023 08:39:52 +0000 Subject: [PATCH 02/10] =?UTF-8?q?update=20profiler/merge=5Fprofiling=5Ftim?= =?UTF-8?q?eline/README.md.=20=E5=88=A0=E9=99=A4=E9=83=A8=E5=88=86?= =?UTF-8?q?=E8=AF=B4=E6=98=8E=EF=BC=8C=E6=9B=B4=E6=96=B0=E8=B6=85=E5=A4=A7?= =?UTF-8?q?timeline=E6=96=87=E4=BB=B6=E6=9F=A5=E7=9C=8B=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/README.md | 27 ++++++++------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/profiler/merge_profiling_timeline/README.md b/profiler/merge_profiling_timeline/README.md index 2d29079570..07989c93ed 100644 --- a/profiler/merge_profiling_timeline/README.md +++ b/profiler/merge_profiling_timeline/README.md @@ -66,18 +66,8 @@ python3 main.py -d path/to/cann_profiling/ --rank 0,1 ``` python3 main.py -d path/to/cann_profiling/ --items CANN,Ascend_Hardware ``` - -4、合并多机多卡的timeline时, 需要-t指定节点间的时间误差文件路径, 用以校准节点间的时间： - -``` -python3 main.py -d path/to/cann_profiling/ -t path/to/time_difference.json --rank 0,8, -``` - -合并timeline查看： -> 在 -o 指定的目录（默认在-d指定的目录下）的msprof_merged_*p.json为合并后的文件 - -## 2 多timeline融合(自定义) -### 2.1 数据采集 +4、合并多timeline(自定义) +4.1 数据采集 将需要合并的timeline文件全部放在同一目录下 数据目录结构示意如下： ``` @@ -92,7 +82,7 @@ python3 main.py -d path/to/cann_profiling/ -t path/to/time_difference.json --ran |- step_trace_3.json ... ``` -### 2.2 合并timeline +4.2 合并timeline 使用脚本`merge_profiling_timeline/main.py`合并timeline 可选参数： @@ -106,14 +96,17 @@ python3 main.py -d path/to/cann_profiling/ -t path/to/time_difference.json --ran ``` python3 main.py -d path/to/timeline/ --type custom ``` -合并timeline查看：同 +合并timeline查看： +> 在 -o 指定的目录（默认在-d指定的目录下）的msprof_merged_*p.json为合并后的文件 + ## 3 超大timeline文件查看 -直接使用以下命令 +下载whl包并安装： +https://gitee.com/aerfaliang/trace_processor/releases/download/trace_processor_37.0/trace_processor-37.0-py3-none-any.whl +安装完成后直接使用以下命令 ``` -cd merge_profiling_timeline -python ./trace_processor --httpd path/to/msprof_merged_*p.json +python3 ./trace_processor --httpd path/to/msprof_merged_*p.json ``` 等待加载完毕，刷新[perfetto](https://ui.perfetto.dev/)界面，点击`YES, use loaded trace`即可展示timeline -- Gitee From f4327871424ca50e96c4b1e9387eb5ffe7ecdb7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Wed, 16 Aug 2023 08:44:17 +0000 Subject: [PATCH 03/10] =?UTF-8?q?update=20profiler/merge=5Fprofiling=5Ftim?= =?UTF-8?q?eline/README.md.=20=E5=88=A0=E9=99=A4-t=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/profiler/merge_profiling_timeline/README.md b/profiler/merge_profiling_timeline/README.md index 07989c93ed..6c9bbb94cc 100644 --- a/profiler/merge_profiling_timeline/README.md +++ b/profiler/merge_profiling_timeline/README.md @@ -37,7 +37,6 @@ ascend pytorch profiling数据目录结构如下 可选参数： - -d: **必选参数**，profiling数据文件或文件夹路径 -- -t: **当需要融合多机多卡timeline时需要校准多机间的时间**，传入时间校准的time_difference.json文件路径， 该文件的获取参考[节点间时间差获取](https://gitee.com/aerfaliang/merge_profiling_timeline/tree/master/get_nodes_timediff) - -o: 可选参数，指定合并后的timeline文件输出的路径，默认为'-d'输入的路径 - --rank：可选参数，指定需要合并timeline的卡号，默认全部合并 - --items：可选参数，指定需要合并的profiling数据项，默认全部合并 @@ -67,6 +66,7 @@ python3 main.py -d path/to/cann_profiling/ --rank 0,1 python3 main.py -d path/to/cann_profiling/ --items CANN,Ascend_Hardware ``` 4、合并多timeline(自定义) + 4.1 数据采集 将需要合并的timeline文件全部放在同一目录下 数据目录结构示意如下： @@ -90,6 +90,7 @@ python3 main.py -d path/to/cann_profiling/ --items CANN,Ascend_Hardware - -o: 可选参数，指定合并后的timeline文件输出的路径，默认为'-d'输入的路径 - --type: 指定需要合并timeline场景，指定参数：`custom` - `custom` ：自定义需要合并的timeline数据 + **使用示例**： 将需要合并的所有timeline放在同一目录下，通过下面的命令合并所有timeline -- Gitee From e0038f644204b208650b5e8025fcb80020bc1c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Wed, 16 Aug 2023 09:21:15 +0000 Subject: [PATCH 04/10] =?UTF-8?q?update=20profiler/merge=5Fprofiling=5Ftim?= =?UTF-8?q?eline/README.md.=20=E5=88=A0=E9=99=A4=E4=B8=8D=E5=BF=85?= =?UTF-8?q?=E8=A6=81=E7=9A=84=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/README.md | 22 ++++++--------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/profiler/merge_profiling_timeline/README.md b/profiler/merge_profiling_timeline/README.md index 6c9bbb94cc..226c4403eb 100644 --- a/profiler/merge_profiling_timeline/README.md +++ b/profiler/merge_profiling_timeline/README.md @@ -9,7 +9,7 @@ ### 1.1 数据采集 使用msporf采集数据，将采集到的所有节点的profiling数据拷贝到当前机器同一目录下，以下假设数据在/home/test/cann_profiling下 -e2e profiling数据目录结构如下, 合并timeline必需数据：`msprof*.json`和`info.json.*`： +e2e profiling数据目录结构如下： ``` |- cann_profiling |- PROF_*** @@ -21,7 +21,7 @@ e2e profiling数据目录结构如下, 合并timeline必需数据：`msprof*.jso |- PROF_*** ... ``` -ascend pytorch profiling数据目录结构如下 +ascend pytorch profiling数据目录结构如下： ``` |- ascend_pytorch_profiling |- **_ascend_pt @@ -43,7 +43,7 @@ ascend pytorch profiling数据目录结构如下 - --type: 指定需要合并timeline场景，可选参数：`pytorch`, `e2e`, `custom` - `pytorch`：通过ascend pytorch方式采集profiling数据，合并所有卡的trace_view.json - `e2e`：通过e2e方式采集profiling数据，优先合并总timeline，没有生成则选择合并device目录下的msprof_*.json - - `custom` ：自定义需要合并的timeline数据，具体参考第2章节 + - `custom` ：自定义需要合并的timeline数据，具体参考示例 @@ -67,9 +67,7 @@ python3 main.py -d path/to/cann_profiling/ --items CANN,Ascend_Hardware ``` 4、合并多timeline(自定义) -4.1 数据采集 -将需要合并的timeline文件全部放在同一目录下 -数据目录结构示意如下： +以上场景不支持的情况下，可以使用自定义的合并方式，将需要合并的timeline文件放在同一目录下，数据目录结构示意如下： ``` |- timeline |- msprof_0.json @@ -82,18 +80,9 @@ python3 main.py -d path/to/cann_profiling/ --items CANN,Ascend_Hardware |- step_trace_3.json ... ``` -4.2 合并timeline -使用脚本`merge_profiling_timeline/main.py`合并timeline - -可选参数： -- -d: **必选参数**，指定profiling数据文件或文件夹路径 -- -o: 可选参数，指定合并后的timeline文件输出的路径，默认为'-d'输入的路径 -- --type: 指定需要合并timeline场景，指定参数：`custom` - - `custom` ：自定义需要合并的timeline数据 - **使用示例**： -将需要合并的所有timeline放在同一目录下，通过下面的命令合并所有timeline +通过下面的命令合并所有timeline，同样支持-o、--rank、--items等参数: ``` python3 main.py -d path/to/timeline/ --type custom ``` @@ -105,6 +94,7 @@ python3 main.py -d path/to/timeline/ --type custom 下载whl包并安装： https://gitee.com/aerfaliang/trace_processor/releases/download/trace_processor_37.0/trace_processor-37.0-py3-none-any.whl + 安装完成后直接使用以下命令 ``` python3 ./trace_processor --httpd path/to/msprof_merged_*p.json -- Gitee From 0de1db4443365e8bb89db3aae7d91f278096e194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Wed, 16 Aug 2023 09:38:01 +0000 Subject: [PATCH 05/10] =?UTF-8?q?update=20profiler/merge=5Fprofiling=5Ftim?= =?UTF-8?q?eline/README.md.=20=E5=88=A0=E9=99=A4=E4=B8=8D=E5=BF=85?= =?UTF-8?q?=E8=A6=81=E7=9A=84=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/README.md | 26 ++++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/profiler/merge_profiling_timeline/README.md b/profiler/merge_profiling_timeline/README.md index 226c4403eb..822cce3e80 100644 --- a/profiler/merge_profiling_timeline/README.md +++ b/profiler/merge_profiling_timeline/README.md @@ -1,15 +1,18 @@ -# Profiling merge tool +# Profiling merge timeline tool ## 介绍 + 本工具支持合并profiling的timeline数据，支持合并指定rank的timline、合并指定timeline中的item ## 1 多timeline融合 ### 1.1 数据采集 + 使用msporf采集数据，将采集到的所有节点的profiling数据拷贝到当前机器同一目录下，以下假设数据在/home/test/cann_profiling下 e2e profiling数据目录结构如下： + ``` |- cann_profiling |- PROF_*** @@ -21,7 +24,9 @@ e2e profiling数据目录结构如下： |- PROF_*** ... ``` -ascend pytorch profiling数据目录结构如下： + +ascend pytorch profiler数据目录结构如下： + ``` |- ascend_pytorch_profiling |- **_ascend_pt @@ -36,6 +41,7 @@ ascend pytorch profiling数据目录结构如下： ### 1.2 合并timeline 可选参数： + - -d: **必选参数**，profiling数据文件或文件夹路径 - -o: 可选参数，指定合并后的timeline文件输出的路径，默认为'-d'输入的路径 - --rank：可选参数，指定需要合并timeline的卡号，默认全部合并 @@ -51,6 +57,7 @@ ascend pytorch profiling数据目录结构如下： **使用示例**： 1、合并单机多卡timeline，默认合并所有卡、所有数据项： + ``` python3 main.py -d path/to/cann_profiling/ ``` @@ -62,12 +69,15 @@ python3 main.py -d path/to/cann_profiling/ --rank 0,1 ``` 3、合并单机多卡timeline，合并所有卡的CANN层和Ascend_Hardware层数据 + ``` python3 main.py -d path/to/cann_profiling/ --items CANN,Ascend_Hardware ``` + 4、合并多timeline(自定义) 以上场景不支持的情况下，可以使用自定义的合并方式，将需要合并的timeline文件放在同一目录下，数据目录结构示意如下： + ``` |- timeline |- msprof_0.json @@ -80,25 +90,29 @@ python3 main.py -d path/to/cann_profiling/ --items CANN,Ascend_Hardware |- step_trace_3.json ... ``` + **使用示例**： 通过下面的命令合并所有timeline，同样支持-o、--rank、--items等参数: + ``` python3 main.py -d path/to/timeline/ --type custom ``` + 合并timeline查看： + > 在 -o 指定的目录（默认在-d指定的目录下）的msprof_merged_*p.json为合并后的文件 -## 3 超大timeline文件查看 +## 2 超大timeline文件查看 -下载whl包并安装： +下载whl包并安装（windows）： https://gitee.com/aerfaliang/trace_processor/releases/download/trace_processor_37.0/trace_processor-37.0-py3-none-any.whl 安装完成后直接使用以下命令 + ``` python3 ./trace_processor --httpd path/to/msprof_merged_*p.json ``` -等待加载完毕，刷新[perfetto](https://ui.perfetto.dev/)界面，点击`YES, use loaded trace`即可展示timeline - +等待加载完毕，刷新[perfetto](https://ui.perfetto.dev/)界面，点击`YES, use loaded trace`即可展示timeline \ No newline at end of file -- Gitee From 6a5fd05f353f168e916555e1e5335a00cdeecbe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Wed, 16 Aug 2023 09:40:56 +0000 Subject: [PATCH 06/10] update profiler/merge_profiling_timeline/main.py. change default value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiler/merge_profiling_timeline/main.py b/profiler/merge_profiling_timeline/main.py index 7e782bff3d..c2b3be9ca1 100644 --- a/profiler/merge_profiling_timeline/main.py +++ b/profiler/merge_profiling_timeline/main.py @@ -106,8 +106,8 @@ def get_absolute_ts_start_info(pro_path) -> float: if start_json: with open(start_json, "r+") as f: info = json.load(f) - ts_us = float(info.get("collectionTimeBegin")) - ts_ns = float(info.get("clockMonotonicRaw")) + ts_us = float(info.get("collectionTimeBegin", 0)) + ts_ns = float(info.get("clockMonotonicRaw", 0)) if not ts_us and not ts_ns: return 0 return ts_us-ts_ns/1000 -- Gitee From 37e71c3333a6a163eb748aa4361fd336fa064229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Sat, 26 Aug 2023 03:05:59 +0000 Subject: [PATCH 07/10] update profiler/merge_profiling_timeline/main.py. list of question change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/main.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/profiler/merge_profiling_timeline/main.py b/profiler/merge_profiling_timeline/main.py index c2b3be9ca1..8186276bf3 100644 --- a/profiler/merge_profiling_timeline/main.py +++ b/profiler/merge_profiling_timeline/main.py @@ -22,7 +22,7 @@ from argparse import ArgumentParser FILTER_DIRS = [".profiler", "HCCL_PROF", "timeline", "query", 'sqlite', 'log'] -MAX_INDEX_COUNT = 1000 +RANK_ID_POS = 1000 # 获取时间差异文件中的node和时间差的对应关系，保存到字典中 @@ -144,6 +144,9 @@ def merge_timeline_general(args): rank_ids = list(timeline_info.keys()) for rank_id in rank_ids: + if not timeline_info.get(rank_id): + print(f"main.py: error rank '{rank_id}' ") + return timeline_files_dict[rank_id] = timeline_info.get(rank_id) merge_timeline_events(timeline_files_dict, process_list) @@ -219,18 +222,21 @@ def merge_timeline_events(timeline_file_dict, process_list): if event.get("args") is not None and event["args"].get("name") is not None: event["args"]["name"] = event["args"]["name"] + f"_{rank_id}" + #modify connect id + if event.get('id') and (event.get('ph') == 's' or event.get('ph') == 'f'): + event['id'] = event.get('id') * RANK_ID_POS + rank_id + new_events.append(event) - output_path = os.path.join(args.output, f"msprof_merged_{len(timeline_file_dict)}p.json") - with open(output_path, 'w') as f: + out_path = f"{args.output}_merged.json" + with open(out_path, 'w') as f: json.dump(new_events, f) - print(f"timeline merged output path: {output_path}") + print(f"timeline merged output path: {out_path}") def parse_args(): parser = ArgumentParser(description="Merge timeline for multi card") parser.add_argument("--data", "-d", default=None, help="root dir of PROF_* data") - parser.add_argument("--timediff", "-t", default=None, help="JSON file for saving startup time differences") parser.add_argument("--output", "-o", default=None, help="save path of msprof_merged.json ") parser.add_argument("--rank", default=None, help="List of ranks to be merged. By default, all ranks are merged") parser.add_argument("--items", default=None, help="Specify the data items to be merged. in the timeline.") -- Gitee From b282e6173b5984bdd8bff8d12dbfa0adebda2a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Sat, 26 Aug 2023 07:34:53 +0000 Subject: [PATCH 08/10] update profiler/merge_profiling_timeline/main.py. change id format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/profiler/merge_profiling_timeline/main.py b/profiler/merge_profiling_timeline/main.py index 8186276bf3..5ac4b89e0c 100644 --- a/profiler/merge_profiling_timeline/main.py +++ b/profiler/merge_profiling_timeline/main.py @@ -145,7 +145,7 @@ def merge_timeline_general(args): for rank_id in rank_ids: if not timeline_info.get(rank_id): - print(f"main.py: error rank '{rank_id}' ") + print(f"main.py: error rank_id '{rank_id}' ") return timeline_files_dict[rank_id] = timeline_info.get(rank_id) merge_timeline_events(timeline_files_dict, process_list) @@ -224,7 +224,7 @@ def merge_timeline_events(timeline_file_dict, process_list): #modify connect id if event.get('id') and (event.get('ph') == 's' or event.get('ph') == 'f'): - event['id'] = event.get('id') * RANK_ID_POS + rank_id + event['id'] = float(event.get('id')) * RANK_ID_POS + rank_id new_events.append(event) @@ -239,7 +239,7 @@ def parse_args(): parser.add_argument("--data", "-d", default=None, help="root dir of PROF_* data") parser.add_argument("--output", "-o", default=None, help="save path of msprof_merged.json ") parser.add_argument("--rank", default=None, help="List of ranks to be merged. By default, all ranks are merged") - parser.add_argument("--items", default=None, help="Specify the data items to be merged. in the timeline.") + parser.add_argument("--items", default=None, help="Specify the data items (python，CANN，Ascend Hardware，HCCL，..)to be merged. in the timeline.") parser.add_argument("--type", choices=('pytorch', 'e2e', 'custom'), help="Customize the timeline file to be merged.") arg = parser.parse_args() return arg -- Gitee From ee734f10907f74dfbbee9c839e20daf4e2651b1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Sat, 26 Aug 2023 07:49:32 +0000 Subject: [PATCH 09/10] =?UTF-8?q?update=20profiler/merge=5Fprofiling=5Ftim?= =?UTF-8?q?eline/README.md.=20=E6=9B=B4=E6=96=B0=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/README.md | 26 ++++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/profiler/merge_profiling_timeline/README.md b/profiler/merge_profiling_timeline/README.md index 822cce3e80..ff0b07a31e 100644 --- a/profiler/merge_profiling_timeline/README.md +++ b/profiler/merge_profiling_timeline/README.md @@ -43,40 +43,41 @@ ascend pytorch profiler数据目录结构如下： 可选参数： - -d: **必选参数**，profiling数据文件或文件夹路径 -- -o: 可选参数，指定合并后的timeline文件输出的路径，默认为'-d'输入的路径 -- --rank：可选参数，指定需要合并timeline的卡号，默认全部合并 -- --items：可选参数，指定需要合并的profiling数据项，默认全部合并 -- --type: 指定需要合并timeline场景，可选参数：`pytorch`, `e2e`, `custom` +- --type: **必选参数**,指定需要合并timeline场景，可选参数有：`pytorch`, `e2e`, `custom` - `pytorch`：通过ascend pytorch方式采集profiling数据，合并所有卡的trace_view.json - `e2e`：通过e2e方式采集profiling数据，优先合并总timeline，没有生成则选择合并device目录下的msprof_*.json - `custom` ：自定义需要合并的timeline数据，具体参考示例 +- -o: 可选参数，指定合并后的timeline文件输出的路径，默认为'-d'输入的路径 +- --rank：可选参数，指定需要合并timeline的卡号，默认全部合并 +- --items：可选参数，指定需要合并的profiling数据项（python，Ascend Hardware，CANN，HCCL，PTA，Overlap Analysis），默认全部合并 **使用示例**： -1、合并单机多卡timeline，默认合并所有卡、所有数据项： +1、合并单机多卡timeline，默认合并所有卡、所有数据项，生成first_merge.json在path/to/cann_profiling/output/目录下(不设置-o参数时默认生成_merge.json在数据目录（path/to/cann_profiling/）下： ``` -python3 main.py -d path/to/cann_profiling/ +python3 main.py -d path/to/cann_profiling/ -o path/to/cann_profiling/output/first --type pytorch ``` 2、合并单机多卡timeline，只合并0卡和1卡： ``` -python3 main.py -d path/to/cann_profiling/ --rank 0,1 +python3 main.py -d path/to/cann_profiling/ -o path/to/cann_profiling/output/2p --type pytorch --rank 0,1 ``` 3、合并单机多卡timeline，合并所有卡的CANN层和Ascend_Hardware层数据 ``` -python3 main.py -d path/to/cann_profiling/ --items CANN,Ascend_Hardware +python3 main.py -d path/to/cann_profiling/ --type pytorch --items CANN,Ascend_Hardware ``` 4、合并多timeline(自定义) -以上场景不支持的情况下，可以使用自定义的合并方式，将需要合并的timeline文件放在同一目录下，数据目录结构示意如下： +以上场景不支持的情况下，可以使用自定义的合并方式，将需要合并的timeline文件放在同一目录下（附：该场景比较特殊，与正常合并不同，无法直接读取info.json中的rank_id, 因此该场景下的rank_id为默认分配的序号，用于区分不同文件的相同层，不代表实际rank_id） +数据目录结构示意如下： ``` |- timeline @@ -101,18 +102,21 @@ python3 main.py -d path/to/timeline/ --type custom 合并timeline查看： -> 在 -o 指定的目录（默认在-d指定的目录下）的msprof_merged_*p.json为合并后的文件 +> 在 -o 指定的目录（默认在-d指定的目录下）的_merged.json为合并后的文件 ## 2 超大timeline文件查看 下载whl包并安装（windows）： https://gitee.com/aerfaliang/trace_processor/releases/download/trace_processor_37.0/trace_processor-37.0-py3-none-any.whl +``` +pip3 install trace_processor-37.0-py3-none-any.whl +``` 安装完成后直接使用以下命令 ``` -python3 ./trace_processor --httpd path/to/msprof_merged_*p.json +python -m trace_processor --httpd path/to/msprof_merged_*p.json ``` 等待加载完毕，刷新[perfetto](https://ui.perfetto.dev/)界面，点击`YES, use loaded trace`即可展示timeline \ No newline at end of file -- Gitee From 865b56db31ac62233e176217b55d55cea6ebd30a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Sat, 26 Aug 2023 08:01:14 +0000 Subject: [PATCH 10/10] =?UTF-8?q?update=20profiler/merge=5Fprofiling=5Ftim?= =?UTF-8?q?eline/README.md.=20=E5=AE=8C=E5=96=84md=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/profiler/merge_profiling_timeline/README.md b/profiler/merge_profiling_timeline/README.md index ff0b07a31e..0f097ca7ae 100644 --- a/profiler/merge_profiling_timeline/README.md +++ b/profiler/merge_profiling_timeline/README.md @@ -47,7 +47,7 @@ ascend pytorch profiler数据目录结构如下： - `pytorch`：通过ascend pytorch方式采集profiling数据，合并所有卡的trace_view.json - `e2e`：通过e2e方式采集profiling数据，优先合并总timeline，没有生成则选择合并device目录下的msprof_*.json - `custom` ：自定义需要合并的timeline数据，具体参考示例 -- -o: 可选参数，指定合并后的timeline文件输出的路径，默认为'-d'输入的路径 +- -o: 可选参数，指定合并后的timeline文件输出的路径（路径末尾可以设置文件名，具体用法参考示例），默认为'-d'输入的路径 - --rank：可选参数，指定需要合并timeline的卡号，默认全部合并 - --items：可选参数，指定需要合并的profiling数据项（python，Ascend Hardware，CANN，HCCL，PTA，Overlap Analysis），默认全部合并 @@ -116,7 +116,7 @@ pip3 install trace_processor-37.0-py3-none-any.whl 安装完成后直接使用以下命令 ``` -python -m trace_processor --httpd path/to/msprof_merged_*p.json +python -m trace_processor --httpd path/to/xxx_merged.json ``` -等待加载完毕，刷新[perfetto](https://ui.perfetto.dev/)界面，点击`YES, use loaded trace`即可展示timeline \ No newline at end of file +等待加载完毕，刷新[perfetto](https://ui.perfetto.dev/)界面，点击Use old version regardless，再点击`YES, use loaded trace`即可展示timeline \ No newline at end of file -- Gitee