From 17547914ceee8f2104f08d2e9eeb6dfd19864f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Wed, 27 Sep 2023 07:34:08 +0000 Subject: [PATCH 1/3] =?UTF-8?q?update=20profiler/merge=5Fprofiling=5Ftimel?= =?UTF-8?q?ine/main.py.=20=E4=BF=AE=E6=94=B9event["ts"]=E8=AE=A1=E7=AE=97?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=BB=A5=E9=80=82=E9=85=8DOverlap=20anaysis?= =?UTF-8?q?=E9=83=A8=E5=88=86=E5=AD=97=E6=AE=B5=E7=9A=84=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/profiler/merge_profiling_timeline/main.py b/profiler/merge_profiling_timeline/main.py index 7fd483a7361..4d242d478cc 100644 --- a/profiler/merge_profiling_timeline/main.py +++ b/profiler/merge_profiling_timeline/main.py @@ -19,6 +19,7 @@ import re from functools import partial from argparse import ArgumentParser +from decimal import Decimal FILTER_DIRS = [".profiler", "HCCL_PROF", "timeline", "query", 'sqlite', 'log'] @@ -190,8 +191,6 @@ def merge_timeline_events(timeline_file_dict, process_list): # 提取待合并的items的pid merged_pids = set() for pro in process_list_tmp: - pro = " ".join(pro.split("_")) if "_" in pro else pro - if pro not in proc_pid_dict.keys(): print(f"main.py: error argument --items: invalid choice: '{pro}' (choose from {list(proc_pid_dict.keys())})") return @@ -205,7 +204,7 @@ def merge_timeline_events(timeline_file_dict, process_list): # 当前节点间时间误差可用时，进行时间校准 if event.get("ts") and ts_difference_us: - event["ts"] = event["ts"] + ts_difference_us + event["ts"] = str(Decimal(str(event["ts"])) + Decimal(str(ts_difference_us))) # 区分不同rank的同一进程的pid if isinstance(event.get("pid"), (str, int)): -- Gitee From d1d8715f8eae99d152ce2348bb00f98544668809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Wed, 27 Sep 2023 07:36:49 +0000 Subject: [PATCH 2/3] =?UTF-8?q?update=20profiler/merge=5Fprofiling=5Ftimel?= =?UTF-8?q?ine/README.md.=20=E6=9B=B4=E6=96=B0=E6=96=87=E6=A1=A3=E6=8F=8F?= =?UTF-8?q?=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/profiler/merge_profiling_timeline/README.md b/profiler/merge_profiling_timeline/README.md index 21e8ca5e26a..85a0f2e4c68 100644 --- a/profiler/merge_profiling_timeline/README.md +++ b/profiler/merge_profiling_timeline/README.md @@ -49,31 +49,31 @@ ascend pytorch profiler数据目录结构如下： - `custom` ：自定义需要合并的timeline数据，具体参考示例 - -o: 可选参数，指定合并后的timeline文件输出的路径（路径末尾可以设置文件名，具体用法参考示例），不设置该参数的情况下默认文件输出的路径为当前目录（默认文件名为merged.json） - --rank：可选参数，指定需要合并timeline的卡号，默认全部合并 -- --items：可选参数，指定需要合并的profiling数据项（python，Ascend_Hardware，CANN，HCCL，PTA，Overlap_Analysis），默认全部合并(item直接使用Ascend Hardware作为参数会被误认为是两个参数，因此作为一个参数时使用'_'连接) +- --items：可选参数，指定需要合并的profiling数据项（python，Ascend Hardware，CANN，HCCL，PTA，Overlap Analysis），默认全部合并 **使用示例**： 1、合并单机多卡timeline，默认合并所有卡、所有数据项，生成first.json在path/to/cann_profiling/output/目录下： ``` -python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/first --type pytorch +python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/first --type "pytorch" ``` 2、合并单机多卡timeline，默认合并所有卡、所有数据项，不设置-o参数时默认生成merge.json在当前目录下： ``` -python3 main.py -i path/to/cann_profiling/ --type pytorch +python3 main.py -i path/to/cann_profiling/ --type "pytorch" ``` 3、合并单机多卡timeline，只合并0卡和1卡： ``` -python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/2p --type pytorch --rank 0,1 +python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/2p --type "pytorch" --rank 0,1 ``` 4、合并单机多卡timeline，合并所有卡的CANN层和Ascend_Hardware层数据 ``` -python3 main.py -i path/to/cann_profiling/ --type pytorch --items CANN,Ascend_Hardware +python3 main.py -i path/to/cann_profiling/ --type pytorch --items "CANN,Ascend Hardware" ``` 5、合并多timeline(自定义) -- Gitee From 606fc0e064a09247b9f7f31fa4874791da98fdf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E6=96=8C?= Date: Wed, 27 Sep 2023 07:39:08 +0000 Subject: [PATCH 3/3] =?UTF-8?q?update=20profiler/merge=5Fprofiling=5Ftimel?= =?UTF-8?q?ine/README.md.=20=E4=BF=AE=E6=94=B9=E6=96=87=E6=A1=A3=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 李斌 --- profiler/merge_profiling_timeline/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/profiler/merge_profiling_timeline/README.md b/profiler/merge_profiling_timeline/README.md index 85a0f2e4c68..4b3162d5b04 100644 --- a/profiler/merge_profiling_timeline/README.md +++ b/profiler/merge_profiling_timeline/README.md @@ -56,18 +56,18 @@ ascend pytorch profiler数据目录结构如下： 1、合并单机多卡timeline，默认合并所有卡、所有数据项，生成first.json在path/to/cann_profiling/output/目录下： ``` -python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/first --type "pytorch" +python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/first --type pytorch ``` 2、合并单机多卡timeline，默认合并所有卡、所有数据项，不设置-o参数时默认生成merge.json在当前目录下： ``` -python3 main.py -i path/to/cann_profiling/ --type "pytorch" +python3 main.py -i path/to/cann_profiling/ --type pytorch ``` 3、合并单机多卡timeline，只合并0卡和1卡： ``` -python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/2p --type "pytorch" --rank 0,1 +python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/2p --type pytorch --rank 0,1 ``` 4、合并单机多卡timeline，合并所有卡的CANN层和Ascend_Hardware层数据 -- Gitee