diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py b/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py index 1d93e77cdc46844365224d6e0f5dcacb71947b43..62190c3548d7db71b478b25bbdc1691842aa47f1 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py +++ b/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_memory_compare.py @@ -10,9 +10,10 @@ from .utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpuEnableMemoryCompare(TestCase): ST_DATA_PATH = os.getenv("MSTT_PROFILER_ST_DATA_PATH", "/home/dcs-50/smoke_project_for_msprof_analyze/mstt_profiler/st_data") - BASE_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_3", "n122-122-067_12380_20240912033946038_ascend_pt") - COMPARISON_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_3", - "n122-122-067_12380_20240912033946038_ascend_pt") + BASE_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_4", + "n122-197-168_1333345_20241105122131111_ascend_pt") + COMPARISON_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_4", + "n122-197-168_1632305_20241105124759292_ascend_pt") OUTPUT_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)), "CompareToolsCmdPytorchNpuVsNpuEnableMemoryCompare") RESULT_EXCEL = "" @@ -37,7 +38,7 @@ class TestCompareToolsCmdPytorchNpuVsNpuEnableMemoryCompare(TestCase): "Base Operator Number", "Comparison Allocated Duration(ms)", "Comparison Allocated Memory(MB)", "Comparison Operator Number", "Diff Memory(MB)", "Diff Ratio"] df = pd.read_excel(self.RESULT_EXCEL, sheet_name="MemoryCompareStatistic", header=2) - self.assertEqual(len(df), 141, msg="pytorch npu vs npu memory compare results 'MemoryCompareStatistic'" + self.assertEqual(len(df), 139, msg="pytorch npu vs npu memory compare results 'MemoryCompareStatistic'" "quantity is wrong") self.assertEqual(headers, df.columns.tolist(), msg="pytorch npu vs npu memory compare results 'MemoryCompareStatistic'" diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py b/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py index 6c2e9392e46cb17747e6c156bdf7e3dfab895d24..416818a97d8f51ac80948caebff906d039a7d43a 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py +++ b/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_operator_compare.py @@ -11,9 +11,10 @@ from .utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpuEnableOperatorCompare(TestCase): ST_DATA_PATH = os.getenv("MSTT_PROFILER_ST_DATA_PATH", "/home/dcs-50/smoke_project_for_msprof_analyze/mstt_profiler/st_data") - BASE_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_3", "n122-122-067_12380_20240912033946038_ascend_pt") - COMPARISON_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_3", - "n122-122-067_12380_20240912033946038_ascend_pt") + BASE_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_4", + "n122-197-168_1333345_20241105122131111_ascend_pt") + COMPARISON_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_4", + "n122-197-168_1632305_20241105124759292_ascend_pt") OUTPUT_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)), "CompareToolsCmdPytorchNpuVsNpuEnableOperatorCompare") RESULT_EXCEL = "" @@ -84,7 +85,7 @@ class TestCompareToolsCmdPytorchNpuVsNpuEnableOperatorCompare(TestCase): headers = ["Top", "Operator Name", "Base Device Duration(ms)", "Base Operator Number", "Comparison Device Duration(ms)", "Comparison Operator Number", "Diff Duration(ms)", "Diff Ratio"] df = pd.read_excel(self.RESULT_EXCEL, sheet_name="OperatorCompareStatistic", header=2) - self.assertEqual(len(df), 141, msg="pytorch npu vs npu operator compare results 'OperatorCompareStatistic'" + self.assertEqual(len(df), 139, msg="pytorch npu vs npu operator compare results 'OperatorCompareStatistic'" "quantity is wrong") self.assertEqual(headers, df.columns.tolist(), msg="pytorch npu vs npu operator compare results " "'OperatorCompareStatistic' headers is wrong") @@ -93,7 +94,7 @@ class TestCompareToolsCmdPytorchNpuVsNpuEnableOperatorCompare(TestCase): headers = ["Top", "Operator Name", "Base Device Duration(ms)", "Base Operator Number", "Comparison Device Duration(ms)", "Comparison Operator Number", "Diff Duration(ms)", "Diff Ratio"] df = pd.read_excel(self.IS_RESULT_EXCEL, sheet_name="OperatorCompareStatistic", header=2) - self.assertEqual(len(df), 141, msg="pytorch npu vs npu use input shape operator compare results " + self.assertEqual(len(df), 139, msg="pytorch npu vs npu use input shape operator compare results " "'OperatorCompareStatistic' quantity is wrong") self.assertEqual(headers, df.columns.tolist(), msg="pytorch npu vs npu use input shape operator compare " "results 'OperatorCompareStatistic' headers is wrong") @@ -102,7 +103,7 @@ class TestCompareToolsCmdPytorchNpuVsNpuEnableOperatorCompare(TestCase): headers = ["Top", "Operator Name", "Base Device Duration(ms)", "Base Operator Number", "Comparison Device Duration(ms)", "Comparison Operator Number", "Diff Duration(ms)", "Diff Ratio"] df = pd.read_excel(self.MKN_RESULT_EXCEL, sheet_name="OperatorCompareStatistic", header=2) - self.assertEqual(len(df), 141, msg="pytorch npu vs npu use input shape operator compare results " + self.assertEqual(len(df), 139, msg="pytorch npu vs npu use input shape operator compare results " "'OperatorCompareStatistic' quantity is wrong") self.assertEqual(headers, df.columns.tolist(), msg="pytorch npu vs npu use input shape operator compare " "results 'OperatorCompareStatistic' headers is wrong") @@ -111,7 +112,7 @@ class TestCompareToolsCmdPytorchNpuVsNpuEnableOperatorCompare(TestCase): headers = ["Top", "Operator Name", "Base Device Duration(ms)", "Base Operator Number", "Comparison Device Duration(ms)", "Comparison Operator Number", "Diff Duration(ms)", "Diff Ratio"] df = pd.read_excel(self.ONM_RESULT_EXCEL, sheet_name="OperatorCompareStatistic", header=2) - self.assertEqual(len(df), 141, msg="pytorch npu vs npu use input shape operator compare results " + self.assertEqual(len(df), 139, msg="pytorch npu vs npu use input shape operator compare results " "'OperatorCompareStatistic' quantity is wrong") self.assertEqual(headers, df.columns.tolist(), msg="pytorch npu vs npu use input shape operator compare " "results 'OperatorCompareStatistic' headers is wrong") diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py b/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py index 97e5f683bda3b02eeab8c332198237fdb57c2421..4e944fac41da5e7a077d5001c8f8c650546efb5b 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py +++ b/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_npu_enable_profiling.py @@ -11,9 +11,10 @@ from .utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpuEnableProfiling(TestCase): ST_DATA_PATH = os.getenv("MSTT_PROFILER_ST_DATA_PATH", "/home/dcs-50/smoke_project_for_msprof_analyze/mstt_profiler/st_data") - BASE_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_3", "n122-122-067_12380_20240912033946038_ascend_pt") - COMPARISON_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_3", - "n122-122-067_12380_20240912033946038_ascend_pt") + BASE_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_4", + "n122-197-168_1333345_20241105122131111_ascend_pt") + COMPARISON_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_4", + "n122-197-168_1632305_20241105124759292_ascend_pt") OUTPUT_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)), "CompareToolsCmdPytorchNpuVsNpuEnableProfiling") RESULT_EXCEL = "" @@ -34,14 +35,13 @@ class TestCompareToolsCmdPytorchNpuVsNpuEnableProfiling(TestCase): def test_overall_metrics(self): duration_exp: List[float] = [ - 14474.86, 1194.01, 1194.01, 10442.62, 10402.34, 40.28, 2821.57, 444.33, 2377.24, 16.47, - 0.18, 23922.06, 4604.98, 9.35, 4595.63, 128.38, 127.35, 1.03, 93.30, 1.27, 92.03, 146.49, 0.10, - 146.39, 23310.81, 23310.81, 170.82, 0.20, 170.62, 373.72, 373.72, 38770.64 + 1725.15, 11.98, 11.98, 31.78, 31.78, 756.62, 705.49, 51.13, 879.83, 66.23, 813.60, 32.90, 12.04, 520.82, + 307.11, 13.12, 293.99, 0.00, 0.00, 207.81, 0.01, 207.80, 5.87, 0.01, 5.86, 0.03, 0.03, 2897.92, 2897.92, + 5143.89 ] - diff_exp: List[float] = [0.37, 0.03, 0.03, 0.27, 0.27, 0.00, 0.07, 0.01, 0.06, 0.00, - 0.00, 0.62, 0.12, 0.00, 0.12, 0.00, 0.00, 0.00, 0.00, 0.00, - 0.00, 0.00, 0.00, 0.00, 0.60, 0.60, 0.00, 0.00, 0.00, 0.01, - 0.01, 1.00] + diff_exp: List[float] = [ + 0.34, 0.00, 0.00, 0.01, 0.01, 0.15, 0.14, 0.01, 0.17, 0.01, 0.16, 0.01, 0.00, 0.10, 0.06, 0.00, 0.06, 0.00, + 0.00, 0.04, 0.00, 0.04, 0.00, 0.00, 0.00, 0.00, 0.00, 0.56, 0.56, 1.00] df = pd.read_excel(self.RESULT_EXCEL, sheet_name="OverallMetrics", header=2) for index, row in df.iterrows(): self.assertEqual(duration_exp[index], round(row["Duration(ms)"], 2), diff --git a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py b/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py index b6830c32958a168221234a638ad086cd957c2681..01dce42c5a6fb77afd36b194f541c014348aad29 100644 --- a/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py +++ b/profiler/test/st/compare_tools/test_compare_tools_cmd_pytorch_npu_vs_npu_step.py @@ -11,9 +11,10 @@ from .utils import execute_cmd, check_result_file class TestCompareToolsCmdPytorchNpuVsNpu(TestCase): ST_DATA_PATH = os.getenv("MSTT_PROFILER_ST_DATA_PATH", "/home/dcs-50/smoke_project_for_msprof_analyze/mstt_profiler/st_data") - BASE_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_3", "n122-122-067_12380_20240912033946038_ascend_pt") - COMPARISON_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_3", - "n122-122-067_12380_20240912033946038_ascend_pt") + BASE_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_4", + "n122-197-168_1333345_20241105122131111_ascend_pt") + COMPARISON_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_4", + "n122-197-168_1632305_20241105124759292_ascend_pt") OUTPUT_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)), "CompareToolsCmdPytorchNpuVsNpuStep") RESULT_EXCEL = "" COMMAND_SUCCESS = 0 @@ -33,9 +34,9 @@ class TestCompareToolsCmdPytorchNpuVsNpu(TestCase): def test_overall_metrics(self): duration_exp = [ - 14474.86, 1194.01, 1194.01, 10442.62, 10402.34, 40.28, 2821.57, 444.33, 2377.24, 16.47, 0.18, 23922.06, - 4604.98, 9.35, 4595.63, 128.38, 127.35, 1.03, 93.30, 1.27, 92.03, 146.49, 0.10, 146.39, 23310.81, 23310.81, - 170.82, 0.20, 170.62, 373.72, 373.72, 38770.64 + 1725.15, 11.98, 11.98, 31.78, 31.78, 756.62, 705.49, 51.13, 879.83, 66.23, 813.60, 32.90, 12.04, 520.82, + 307.11, 13.12, 293.99, 0.00, 0.00, 207.81, 0.01, 207.80, 5.87, 0.01, 5.86, 0.03, 0.03, 2897.92, 2897.92, + 5143.89 ] df = pd.read_excel(self.RESULT_EXCEL, sheet_name="OverallMetrics", header=2) for index, row in df.iterrows(): @@ -49,14 +50,13 @@ class TestCompareToolsCmdPytorchNpuVsNpu(TestCase): "Min Duration(us)", "Calls", "Total Duration(us).1", "Avg Duration(us).1", "Max Duration(us).1", "Min Duration(us).1", "Calls.1", "Diff Total Ratio", "Diff Avg Ratio"] df = pd.read_excel(self.RESULT_EXCEL, sheet_name="KernelCompare", header=2) - self.assertEqual(len(df), 703, msg="pytorch npu vs npu step compare results quantity is wrong") + self.assertEqual(len(df), 488, msg="pytorch npu vs npu step compare results quantity is wrong") self.assertEqual(headers, df.columns.tolist(), msg="pytorch npu vs npu step compare results headers is wrong") def test_communication_compare(self): total_duration: List[float] = [ - 9354.86, 1046.68, 9191.52, 24.74, 27743477.86, 12418099.90, 23832.90, 28928712.46, 18411.66, 2939703.28, - 327934.12, 17074.96, 77.58, 931489.92, 2894.42, 75.46, 80.86, 15119087.00, 3594561.44, 12963.36, - 12692002.20, 6180907.46, 9859.70 + 351054.85, 7.22, 400355.22, 80.52, 590652.54, 8.96, 25518.15, 67.62, 49.08, 389.39, 41357.87, 15.68, + 80.18, 144247.88, 867518.01, 4973.28, 336.97, 91039.10, 30.74 ] df = pd.read_excel(self.RESULT_EXCEL, sheet_name="CommunicationCompare", header=2) for index, row in df.iterrows(): @@ -68,7 +68,7 @@ class TestCompareToolsCmdPytorchNpuVsNpu(TestCase): headers = ["Top", "Operator Name", "Base Device Duration(ms)", "Base Operator Number", "Comparison Device Duration(ms)", "Comparison Operator Number", "Diff Duration(ms)", "Diff Ratio"] df = pd.read_excel(self.RESULT_EXCEL, sheet_name="OperatorCompareStatistic", header=2) - self.assertEqual(len(df), 141, msg="pytorch npu vs npu operator compare results 'OperatorCompareStatistic'" + self.assertEqual(len(df), 139, msg="pytorch npu vs npu operator compare results 'OperatorCompareStatistic'" "quantity is wrong") self.assertEqual(headers, df.columns.tolist(), msg="pytorch npu vs npu operator compare results 'OperatorCompareStatistic'" @@ -79,7 +79,7 @@ class TestCompareToolsCmdPytorchNpuVsNpu(TestCase): "Base Operator Number", "Comparison Allocated Duration(ms)", "Comparison Allocated Memory(MB)", "Comparison Operator Number", "Diff Memory(MB)", "Diff Ratio"] df = pd.read_excel(self.RESULT_EXCEL, sheet_name="MemoryCompareStatistic", header=2) - self.assertEqual(len(df), 141, msg="pytorch npu vs npu memory compare results 'MemoryCompareStatistic'" + self.assertEqual(len(df), 139, msg="pytorch npu vs npu memory compare results 'MemoryCompareStatistic'" "quantity is wrong") self.assertEqual(headers, df.columns.tolist(), msg="pytorch npu vs npu memory compare results " "'MemoryCompareStatistic' headers is wrong") @@ -89,5 +89,5 @@ class TestCompareToolsCmdPytorchNpuVsNpu(TestCase): "Calls", "Total Duration(ms).1", "Self Time(ms).1", "Avg Duration(ms).1", "Calls.1", "Diff Total Ratio", "Diff Self Ratio", "Diff Avg Ratio", "Diff Calls Ratio"] df = pd.read_excel(self.RESULT_EXCEL, sheet_name="ApiCompare", header=2) - self.assertEqual(len(df), 311, msg="pytorch npu vs npu api compare results quantity is wrong") + self.assertEqual(len(df), 310, msg="pytorch npu vs npu api compare results quantity is wrong") self.assertEqual(headers, df.columns.tolist(), msg="pytorch npu vs npu api compare results headers is wrong")