From c3d0af716ce6faab455127bca8b9c150788dd440 Mon Sep 17 00:00:00 2001 From: huxianglong Date: Wed, 12 Feb 2025 14:32:53 +0800 Subject: [PATCH 1/4] st --- .../st/tinker/profiled_data_comparison.py | 125 ++++++++++++ .../test/st/tinker/test_gen_modellink_plan.py | 184 ++++++++++++++++++ .../test/st/tinker/test_profiler_space.py | 148 ++++++++++++++ 3 files changed, 457 insertions(+) create mode 100644 profiler/msprof_analyze/test/st/tinker/profiled_data_comparison.py create mode 100644 profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py create mode 100644 profiler/msprof_analyze/test/st/tinker/test_profiler_space.py diff --git a/profiler/msprof_analyze/test/st/tinker/profiled_data_comparison.py b/profiler/msprof_analyze/test/st/tinker/profiled_data_comparison.py new file mode 100644 index 0000000000..d8a878db8a --- /dev/null +++ b/profiler/msprof_analyze/test/st/tinker/profiled_data_comparison.py @@ -0,0 +1,125 @@ +# Copyright (c) 2023, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import glob +import csv +import sys +from pathlib import Path + +sys.path.append("/") +from profiler.msprof_analyze.tinker.utils.utils import project_root + +ST_DATA_PATH = os.getenv("MSTT_PROFILER_ST_DATA_PATH", + "/home/dcs-50/smoke_project_for_msprof_analyze/mstt_profiler/st_data/tinker_data") + + +def _count_csv_files(directory): + """ + 获取文件夹中除去以node.csv为结尾的.csv文件个数 + Returns: + """ + csv_files = glob.glob(os.path.join(directory, '*.csv')) + node_files = glob.glob(os.path.join(directory, '*node.csv')) + return len(csv_files) - len(node_files) + + +def _write_csv_data(folder_path1, folder_path2, op_file_path): + """ + 写入csv数据 + Returns: + """ + # 获取两个文件夹中除去以node.csv为结尾的.csv文件个数 + folder1_count = _count_csv_files(folder_path1) + folder2_count = _count_csv_files(folder_path2) + + # 获取两个文件夹中相同名称的.csv文件路径，排序增加可读性 + files1 = set(file for file in os.listdir(folder_path1) if file.endswith('.csv')) + files2 = set(file for file in os.listdir(folder_path2) if file.endswith('.csv')) + common_files = files1.intersection(files2) + miss_files = sorted(files1 - common_files) + common_files = sorted(common_files) + + # 写入.csv文件生成个数、相同名称个数、标准生成个数、缺失的profiled数据 + with open(op_file_path, 'a', newline='') as output_file: + output_writer = csv.writer(output_file) + output_writer.writerow(['文件生成个数', '相同名称个数', '标准生成个数']) + output_writer.writerow([folder2_count, len(common_files), folder1_count]) + output_writer.writerow([]) + output_writer.writerow(['缺失的profiled']) + for file_name in miss_files: + output_writer.writerow([file_name[:-4]]) + output_writer.writerow([]) + output_writer.writerow(['相同的profiled']) + + # 遍历相同的csv文件，对比每一项内容的差值 + for file_name in common_files: + file_path1 = os.path.join(folder_path1, file_name) + file_path2 = os.path.join(folder_path2, file_name) + + with open(file_path1, 'r') as f1, open(file_path2, 'r') as f2, open(op_file_path, 'a', + newline='') as output_file: + input_file_reader1 = csv.reader(f1) + input_file_reader2 = csv.reader(f2) + output_writer = csv.writer(output_file) + + # 读取第一行并跳过 + header1 = next(input_file_reader1) + next(input_file_reader2) + + # 写入并行特征及两个新表头 + output_writer.writerow([file_name[:-4]] + [''] * 9 + ['标准数据'] + [''] * 9 + ['待测数据']) + + # 写入原先第一行的表头 + table_head = [header1[0]] + header1[1:] + [''] + output_writer.writerow(table_head * 3) + + # 计算每一行内容的差值比率或绝对差值，将其与标准数据、原始数据一同写入 + for row1, row2 in zip(input_file_reader1, input_file_reader2): + diff_row = [row1[0]] + for i in range(1, len(row1)): + if float(row1[i]) != 0: + comparison_value = (float(row1[i]) - float(row2[i])) / float(row1[i]) + diff_row.append(f"{comparison_value * 100:.2f}%") + elif float(row2[i]) != 0: + diff_row.append(float(row1[i]) - float(row2[i])) + else: + diff_row.append(f"{0:.2f}%") + output_writer.writerow(diff_row + [''] + row1 + [''] + row2) + + # 写入空行，方便表格的查看 + output_writer.writerow([]) + + +def main(model_name, model_size, profiler_data_path): + """对比不同文件夹下csv的个数、相同个数、相同参数数值之间的差值，并写入新的csv文件中""" + # 对比文件夹路径 + standard_path = os.path.join(ST_DATA_PATH, 'standard_profiled_data/' + model_name + '-' + model_size) + + # 定义输出文件的路径，若已有文件则删除 + output_file_path = os.path.join(profiler_data_path, 'comparison_value.csv') + if Path(output_file_path).exists(): + os.remove(output_file_path) + + # 创建输出文件 + Path(output_file_path).parent.mkdir(parents=True, exist_ok=True) + + # 写入数据 + _write_csv_data(standard_path, profiler_data_path, output_file_path) + + +if __name__ == '__main__': + # 三个参数：模型名、参数大小、要进行对比的profiled_data的路径 + main(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py b/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py new file mode 100644 index 0000000000..83fb01707b --- /dev/null +++ b/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py @@ -0,0 +1,184 @@ +# Copyright (c) 2023, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing +import os +import re +import shutil +import sys +import unittest +import uuid +from collections import namedtuple +from unittest import mock + +sys.path.append("/") + +from profiler.msprof_analyze.tinker.utils.utils import project_root, read_file +from profiler.msprof_analyze.tinker.search import gen_modellink_plan + +Strategy = namedtuple('Strategy', ['tp', 'pp', 'dp', 'sp', 'zero', 'mbs', 'num_layer_list', 'rc']) +Performance = namedtuple('Performance', ['time_cost', 'mem_cost', 'time_costs', 'mem_costs', 'token_per_npu_sec']) +ST_DATA_PATH = os.getenv("MSTT_PROFILER_ST_DATA_PATH", + "/home/dcs-50/smoke_project_for_msprof_analyze/mstt_profiler/st_data/tinker_data") + + +class TestGenModellinkPlan(unittest.TestCase): + @classmethod + def tearDownClass(cls): + """ + 关闭作用域 + :return: + """ + mock.patch.stopall() + + @staticmethod + def _convert_to_obj(data_dict): + """ + 拆分字典为对象 + :param data_dict: + :return: + """ + strategy_attr_list = getattr(Strategy, '_fields') + strategy_dict = {} + for strategy_attr in strategy_attr_list: + strategy_dict[strategy_attr] = data_dict.get(strategy_attr, None) + strategy = Strategy(**strategy_dict) + performance_attr_list = getattr(Performance, '_fields') + performance_dict = {} + for performance_attr in performance_attr_list: + performance_dict[performance_attr] = data_dict.get(performance_attr, None) + performance = Performance(**performance_dict) + return performance, strategy + + @staticmethod + def _convert_to_dict(line): + """ + 解析行内容，形成字典 + :param line: + :return: + """ + striped_line = line.strip() + pattern = r'([^,\s]+)\s*=\s*([^A-Za-z]+)' + matches = re.findall(pattern, striped_line) + # 将提取结果转换为字典 + data_dict = {key.strip(): value.rstrip(', ') for key, value in matches} + # token/npu/sec 不能作为属性的命名，故需做一道转换 + convert_pair = ('token/npu/sec', 'token_per_npu_sec') + org_value = data_dict.get(convert_pair[0], None) + del data_dict[convert_pair[0]] + data_dict[convert_pair[1]] = org_value + return data_dict + + def test_search_result(self): + """ + 端到端测试search结果 + :return: + """ + # 模拟命令行入参 + cpu_count = multiprocessing.cpu_count() + cpus = 10 if cpu_count is None else cpu_count + unique_id = uuid.uuid1() + save_path = f'./temp_{unique_id}' + sys.argv = ['gen_modellink_plan.py', '--model-name', 'llama2', '--model-size', '7b', '--global-batch-size', + '32', '--num-nodes', '1', '--num-npus-per-node', '8', + '--memory-limit', '56000', '--profiled-data-path', + '../standard_profiled_data/llama2-7b', + '--cpus', f'{cpus}', '--profiled-data-path', + os.path.join(ST_DATA_PATH, 'standard_profiled_data', 'llama2-7b'), '--output-dir', save_path] + + patch = mock.patch('tinker.search.gen_modellink_plan.print_args').start() + gen_modellink_plan.main() + log_file_path = patch.call_args[0][0].log_file + log_output = read_file(log_file_path) + + if os.path.exists(save_path): + # 删除文件夹及其所有内容 + shutil.rmtree(save_path) + + # 获取实测 + self.assertIsNotNone(log_output) + filtered_experimental_data = self._filter_records(log_output.splitlines()) + + # 获取标准输出文件 + + standard_output_file = os.path.join(ST_DATA_PATH, 'standard_search_data/llama2-7b-result.txt') + with open(standard_output_file, 'r', encoding='utf-8') as file: + standard_output = file.readlines() + self.assertIsNotNone(standard_output) + filtered_standard_output = self._filter_records(standard_output) + + # 转为字典 + standard_output_dict, standard_best = self._process_metrics(filtered_standard_output) + experimental_data_dict, experimental_best = self._process_metrics(filtered_experimental_data) + + self.assertEqual(len(standard_output_dict), len(experimental_data_dict), '结果数量应一致') + self.assertEqual(standard_best, experimental_best, '标准最佳结果 != 测试的最佳结果') + + for key, value in experimental_data_dict.items(): + self.assertEqual(standard_output_dict.get(key, None), value, msg='标准结果 != gen_modellink_plan 实际结果。') + + def _filter_records(self, printed_output: list): + """ + 打印结果过滤掉无关信息 + :param printed_output: + :return: + """ + start_idx = None + for i, line in enumerate(printed_output): + if line.startswith('token/npu/sec ='): + start_idx = i + break + self.assertIsNotNone(start_idx) + + end_idx = None + for j in range(start_idx, len(printed_output)): + if printed_output[j].startswith('[TOTAL TIME]'): + end_idx = j + break + + self.assertIsNotNone(end_idx) + + res = printed_output[start_idx: end_idx] + return res + + def _process_metrics(self, content: list): + """ + 提取 search结果，转为 Strategy -> Performance 形式的dict + :param content: + :return: + """ + all_results = {} + best_result = None + best_idx = None + for i, line in enumerate(content): + # 打印结果，以 Best结束 + if 'Best' in line: + best_idx = i + next_line = content[i + 1] + best_data = TestGenModellinkPlan._convert_to_dict(next_line) + performance, strategy = TestGenModellinkPlan._convert_to_obj(best_data) + best_result = (performance, strategy) + break + # 使用正则表达式提取键值对 + data_dict = TestGenModellinkPlan._convert_to_dict(line) + + # clean code 要求 + performance, strategy = TestGenModellinkPlan._convert_to_obj(data_dict) + + all_results[strategy] = performance + self.assertIsNotNone(best_idx) + # best_idx 的含义是所有未去重的并行策略；all_results则是去重后的并行策略 + self.assertEqual(best_idx, len(all_results), '结果中存在重复的并行策略！') + return all_results, best_result diff --git a/profiler/msprof_analyze/test/st/tinker/test_profiler_space.py b/profiler/msprof_analyze/test/st/tinker/test_profiler_space.py new file mode 100644 index 0000000000..30787d935f --- /dev/null +++ b/profiler/msprof_analyze/test/st/tinker/test_profiler_space.py @@ -0,0 +1,148 @@ +# Copyright (c) 2023, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import sys +import os +import csv +import shutil + +sys.path.append("/") +import profiled_data_comparison +from profiler.msprof_analyze.tinker.utils.utils import project_root +from profiler.msprof_analyze.tinker.profiler import profile_space +from profiler.msprof_analyze.tinker.profiler import gen_model_structure + +ST_DATA_PATH = os.getenv("MSTT_PROFILER_ST_DATA_PATH", + "/home/dcs-50/smoke_project_for_msprof_analyze/mstt_profiler/st_data/tinker_data") +PROFILE_DATA_PATH = os.path.join(ST_DATA_PATH, "test_profiled_data") + + +def clear_test_data(test_save_path): + """清空过往测试数据""" + if os.path.exists(test_save_path): + shutil.rmtree(test_save_path) + + +def is_value_numeric(content_str): + """判断值是否为数字""" + try: + float(content_str) + return True + except ValueError: + return False + + +class BaseTestCase(unittest.TestCase): + def validate_comparison_values(self, model_name, reader): + """验证比较值是否在允许的范围内""" + parallel_feature = '' + model_table_header = [] + for row in reader: + if len(row) < 9: + continue + if row[0].startswith(model_name): + parallel_feature = row[0] + if row[0].startswith("block_name"): + model_table_header = row[:9] + for index, fruit in enumerate(row): + if fruit.endswith('%'): + num_value = float(fruit[:-1]) + self.assertTrue(abs(num_value) <= 10, + f'并行特征为{parallel_feature}, {row[0]}的{model_table_header[index]}值的差值为{fruit}') + elif is_value_numeric(fruit): + self.assertTrue((index == 7 or index == 8) and abs(float(fruit)) <= 40, + f'并行特征为{parallel_feature}, {row[0]}的{model_table_header[index]}值的差值为{fruit}') + + +class TestSectionProfilerSpace(BaseTestCase): + @classmethod + def setUpClass(cls): + clear_test_data(PROFILE_DATA_PATH) + + def section_csv_value_difference(self, model_name, model_size): + """部分测试，验证比较值是否在允许的范围内""" + sys.argv = ['gen_model_structure', '-m', model_name, '-s', model_size, + '-v', 'modellink-1.1', '-d', PROFILE_DATA_PATH, '-o'] + gen_model_structure.main() + sys.argv.clear() + sys.argv = ['profile_space', model_name, model_size, '--tp', '4', '--save-path', PROFILE_DATA_PATH] + profile_path = profile_space.main() + comparison_file = os.path.join(ST_DATA_PATH, 'comparison_value.csv') + profiled_data_comparison.main(model_name, model_size, profile_path) + with open(comparison_file, newline='') as csvfile: + reader = csv.reader(csvfile) + self.validate_comparison_values(model_name, reader) + + def test_qwen15_32b_csv_value_difference(self): + """测试qwen15-32B""" + self.section_csv_value_difference('qwen15', '32b') + + def test_qwen15_7b_csv_value_difference(self): + """测试qwen15-7B""" + self.section_csv_value_difference('qwen15', '7b') + + def test_codellama_34b_csv_value_difference(self): + """测试codellama-34B""" + self.section_csv_value_difference('codellama', '34b') + + def test_chatglm3_6b_csv_value_difference(self): + """测试chatglm3-6B""" + self.section_csv_value_difference('chatglm3', '6b') + + +class TestProfilerSpace(BaseTestCase): + @classmethod + def setUpClass(cls): + clear_test_data(PROFILE_DATA_PATH) + cls.standard_path = os.path.join(ST_DATA_PATH, "standard_profiled_data") + cls.comparison_paths = [] + cls.standard_names = [] + for standard_name in os.listdir(cls.standard_path): + if '-' not in standard_name: + continue + if os.path.isdir(os.path.join(cls.standard_path, standard_name)): + cls.standard_names.append(standard_name) + model_parameters = standard_name.split('-') + sys.argv = ['gen_model_structure', '-m', model_parameters[0], '-s', model_parameters[1], + '-v', 'modellink-1.1', '-d', PROFILE_DATA_PATH, '-o'] + gen_model_structure.main() + sys.argv.clear() + sys.argv = ['profile_space', model_parameters[0], model_parameters[1], '--save-path', + PROFILE_DATA_PATH] + profile_path = profile_space.main() + cls.comparison_paths.append(os.path.join(profile_path, 'comparison_value.csv')) + profiled_data_comparison.main(model_parameters[0], model_parameters[1], profile_path) + + def test_csv_num_same(self): + """全量测试文件生成个数、相同名称个数、标准生成个数是否都相同""" + for index, comparison_file in enumerate(self.comparison_paths): + with open(comparison_file, newline='') as csvfile: + reader = csv.reader(csvfile) + next(reader) + comparison_nums = next(reader)[:3] + self.assertEqual(comparison_nums[0], comparison_nums[2], + f'{self.standard_names[index]}的文件生成个数与标准生成个数不同') + self.assertEqual(comparison_nums[1], comparison_nums[2], + f'{self.standard_names[index]}的相同名称个数与标准生成个数不同') + + def test_csv_value_difference(self): + """全量测试，验证比较值是否在允许的范围内""" + for index, comparison_file in enumerate(self.comparison_paths): + model_parameters = self.standard_names[index].split('-') + model_name = model_parameters[0] + with open(comparison_file, newline='') as csvfile: + reader = csv.reader(csvfile) + self.validate_comparison_values(model_name, reader) -- Gitee From ed033fba19a42c36cc93b81d297337cf4ba52f37 Mon Sep 17 00:00:00 2001 From: huxianglong Date: Wed, 12 Feb 2025 14:36:15 +0800 Subject: [PATCH 2/4] st --- .../msprof_analyze/test/st/tinker/test_gen_modellink_plan.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py b/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py index 83fb01707b..b1b4da6a92 100644 --- a/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py +++ b/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py @@ -35,6 +35,9 @@ ST_DATA_PATH = os.getenv("MSTT_PROFILER_ST_DATA_PATH", class TestGenModellinkPlan(unittest.TestCase): + def __init__(self,*args,**kwargs): + super(TestGenModellinkPlan,self).__init__(*args,**kwargs) + @classmethod def tearDownClass(cls): """ -- Gitee From d95c01a60b6b1b7db30a9f6de5fb4b5237f04b3a Mon Sep 17 00:00:00 2001 From: huxianglong Date: Wed, 12 Feb 2025 14:47:50 +0800 Subject: [PATCH 3/4] st --- .../test/st/tinker/test_gen_modellink_plan.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py b/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py index b1b4da6a92..0194d35ede 100644 --- a/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py +++ b/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py @@ -35,17 +35,6 @@ ST_DATA_PATH = os.getenv("MSTT_PROFILER_ST_DATA_PATH", class TestGenModellinkPlan(unittest.TestCase): - def __init__(self,*args,**kwargs): - super(TestGenModellinkPlan,self).__init__(*args,**kwargs) - - @classmethod - def tearDownClass(cls): - """ - 关闭作用域 - :return: - """ - mock.patch.stopall() - @staticmethod def _convert_to_obj(data_dict): """ @@ -84,6 +73,14 @@ class TestGenModellinkPlan(unittest.TestCase): data_dict[convert_pair[1]] = org_value return data_dict + @classmethod + def tearDownClass(cls): + """ + 关闭作用域 + :return: + """ + mock.patch.stopall() + def test_search_result(self): """ 端到端测试search结果 -- Gitee From a4d707429cbdc39682b38b6d76d04690c42f6142 Mon Sep 17 00:00:00 2001 From: huxianglong Date: Wed, 12 Feb 2025 16:18:14 +0800 Subject: [PATCH 4/4] st --- .../test/st/tinker/test_gen_modellink_plan.py | 4 +++- .../msprof_analyze/test/st/tinker/test_profiler_space.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py b/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py index 0194d35ede..79928b85be 100644 --- a/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py +++ b/profiler/msprof_analyze/test/st/tinker/test_gen_modellink_plan.py @@ -23,6 +23,8 @@ import uuid from collections import namedtuple from unittest import mock +from profiler.msprof_analyze.tinker.utils.config import parse_args + sys.path.append("/") from profiler.msprof_analyze.tinker.utils.utils import project_root, read_file @@ -99,7 +101,7 @@ class TestGenModellinkPlan(unittest.TestCase): os.path.join(ST_DATA_PATH, 'standard_profiled_data', 'llama2-7b'), '--output-dir', save_path] patch = mock.patch('tinker.search.gen_modellink_plan.print_args').start() - gen_modellink_plan.main() + gen_modellink_plan.run(parse_args()) log_file_path = patch.call_args[0][0].log_file log_output = read_file(log_file_path) diff --git a/profiler/msprof_analyze/test/st/tinker/test_profiler_space.py b/profiler/msprof_analyze/test/st/tinker/test_profiler_space.py index 30787d935f..8e8c064c91 100644 --- a/profiler/msprof_analyze/test/st/tinker/test_profiler_space.py +++ b/profiler/msprof_analyze/test/st/tinker/test_profiler_space.py @@ -19,6 +19,8 @@ import os import csv import shutil +from profiler.msprof_analyze.tinker.utils.config import parse_args + sys.path.append("/") import profiled_data_comparison from profiler.msprof_analyze.tinker.utils.utils import project_root @@ -79,7 +81,7 @@ class TestSectionProfilerSpace(BaseTestCase): gen_model_structure.main() sys.argv.clear() sys.argv = ['profile_space', model_name, model_size, '--tp', '4', '--save-path', PROFILE_DATA_PATH] - profile_path = profile_space.main() + profile_path = profile_space.run(parse_args()) comparison_file = os.path.join(ST_DATA_PATH, 'comparison_value.csv') profiled_data_comparison.main(model_name, model_size, profile_path) with open(comparison_file, newline='') as csvfile: @@ -122,7 +124,7 @@ class TestProfilerSpace(BaseTestCase): sys.argv.clear() sys.argv = ['profile_space', model_parameters[0], model_parameters[1], '--save-path', PROFILE_DATA_PATH] - profile_path = profile_space.main() + profile_path = profile_space.run(parse_args()) cls.comparison_paths.append(os.path.join(profile_path, 'comparison_value.csv')) profiled_data_comparison.main(model_parameters[0], model_parameters[1], profile_path) -- Gitee