diff --git a/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py b/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py index 1f306415fa789ae0dab7d8751b1c240b3433de0d..7bb49fd5c56f7c782f2f90abe26d63ecf51e165b 100644 --- a/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py +++ b/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import copy from common_func.constant import Constant @@ -5,20 +20,32 @@ from common_func.table_constant import TableConstant class DataTransferAdapter(object): - COMM_TIME_TABLE_COLUMN = [TableConstant.START_TIMESTAMP, TableConstant.ELAPSED_TIME, TableConstant.TRANSIT_TIME, - TableConstant.WAIT_TIME, TableConstant.SYNCHRONIZATION_TIME, TableConstant.IDLE_TIME, - TableConstant.SYNCHRONIZATION_TIME_RATIO, TableConstant.WAIT_TIME_RATIO] - COMM_TIME_JSON_COLUMN = [Constant.START_TIMESTAMP, Constant.ELAPSE_TIME_MS, Constant.TRANSIT_TIME_MS, - Constant.WAIT_TIME_MS, Constant.SYNCHRONIZATION_TIME_MS, Constant.IDLE_TIME_MS, - Constant.SYNCHRONIZATION_TIME_RATIO, Constant.WAIT_TIME_RATIO] - MATRIX_TABLE_COLUMN = [TableConstant.TRANSIT_SIZE, TableConstant.TRANSIT_TIME, TableConstant.BANDWIDTH, - TableConstant.TRANSPORT_TYPE, TableConstant.OPNAME] - MATRIX_JSON_COLUMN = [Constant.TRANSIT_SIZE_MB, Constant.TRANSIT_TIME_MS, Constant.BANDWIDTH_GB_S, - Constant.TRANSPORT_TYPE, Constant.OP_NAME] - COMM_BD_TABLE_COLUMN = [TableConstant.TRANSIT_SIZE, TableConstant.TRANSIT_TIME, TableConstant.BANDWIDTH, - TableConstant.LARGE_PACKET_RATIO] - COMM_BD_JSON_COLUMN = [Constant.TRANSIT_SIZE_MB, Constant.TRANSIT_TIME_MS, Constant.BANDWIDTH_GB_S, - Constant.LARGE_PACKET_RATIO] + COMM_TIME_TABLE_COLUMN = [ + TableConstant.START_TIMESTAMP, TableConstant.ELAPSED_TIME, TableConstant.TRANSIT_TIME, + TableConstant.WAIT_TIME, TableConstant.SYNCHRONIZATION_TIME, TableConstant.IDLE_TIME, + TableConstant.SYNCHRONIZATION_TIME_RATIO, TableConstant.WAIT_TIME_RATIO, + ] + COMM_TIME_JSON_COLUMN = [ + Constant.START_TIMESTAMP, Constant.ELAPSE_TIME_MS, Constant.TRANSIT_TIME_MS, + Constant.WAIT_TIME_MS, Constant.SYNCHRONIZATION_TIME_MS, Constant.IDLE_TIME_MS, + Constant.SYNCHRONIZATION_TIME_RATIO, Constant.WAIT_TIME_RATIO, + ] + MATRIX_TABLE_COLUMN = [ + TableConstant.TRANSIT_SIZE, TableConstant.TRANSIT_TIME, TableConstant.BANDWIDTH, + TableConstant.TRANSPORT_TYPE, TableConstant.OPNAME, + ] + MATRIX_JSON_COLUMN = [ + Constant.TRANSIT_SIZE_MB, Constant.TRANSIT_TIME_MS, Constant.BANDWIDTH_GB_S, + Constant.TRANSPORT_TYPE, Constant.OP_NAME, + ] + COMM_BD_TABLE_COLUMN = [ + TableConstant.TRANSIT_SIZE, TableConstant.TRANSIT_TIME, TableConstant.BANDWIDTH, + TableConstant.LARGE_PACKET_RATIO, + ] + COMM_BD_JSON_COLUMN = [ + Constant.TRANSIT_SIZE_MB, Constant.TRANSIT_TIME_MS, Constant.BANDWIDTH_GB_S, + Constant.LARGE_PACKET_RATIO, + ] def __init__(self): super().__init__() diff --git a/profiler/cluster_analyse/common_func/db_manager.py b/profiler/cluster_analyse/common_func/db_manager.py index 1aa7ed8740e4baa6fd5f04ec1674b20d584517c3..b19953f59df05974d8a53eac5e451def3229df3f 100644 --- a/profiler/cluster_analyse/common_func/db_manager.py +++ b/profiler/cluster_analyse/common_func/db_manager.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os import sqlite3 @@ -21,6 +22,7 @@ from common_func.empty_class import EmptyClass from common_func.file_manager import check_db_path_valid from common_func.tables_config import TablesConfig + class DBManager: """ class to manage DB operation @@ -38,7 +40,7 @@ class DBManager: try: conn = sqlite3.connect(db_path) except sqlite3.Error as err: - print(f"[ERROR] {err}") + logging.error(err) return EmptyClass("empty conn"), EmptyClass("empty curs") try: if isinstance(conn, sqlite3.Connection): @@ -46,7 +48,7 @@ class DBManager: os.chmod(db_path, Constant.FILE_AUTHORITY) return conn, curs except sqlite3.Error as err: - print(f"[ERROR] {err}") + logging.error(err) return EmptyClass("empty conn"), EmptyClass("empty curs") return EmptyClass("empty conn"), EmptyClass("empty curs") @@ -59,12 +61,12 @@ class DBManager: if isinstance(curs, sqlite3.Cursor): curs.close() except sqlite3.Error as err: - print(f"[ERROR] {err}") + logging.error(err) try: if isinstance(conn, sqlite3.Connection): conn.close() except sqlite3.Error as err: - print(f"[ERROR] {err}") + logging.error(err) @staticmethod def judge_table_exists(curs: any, table_name: str) -> any: @@ -77,7 +79,7 @@ class DBManager: curs.execute("select count(*) from sqlite_master where type='table' and name=?", (table_name,)) return curs.fetchone()[0] except sqlite3.Error as err: - print("[ERROR] {}".format(err)) + logging.error(err) return False @staticmethod @@ -97,6 +99,41 @@ class DBManager: return header_with_type_begin return "" + @staticmethod + def execute_sql(conn: any, sql: str, params: any = None) -> bool: + """ + execute sql + """ + try: + if isinstance(conn, sqlite3.Connection): + if params: + conn.cursor().execute(sql, params) + else: + conn.cursor().execute(sql) + conn.commit() + return True + except sqlite3.Error as err: + logging.error(err) + return False + logging.error("conn is invalid param") + return False + + @staticmethod + def executemany_sql(conn: any, sql: str, params: any) -> bool: + """ + execute many sql once + """ + try: + if isinstance(conn, sqlite3.Connection): + conn.cursor().executemany(sql, params) + conn.commit() + return True + except sqlite3.Error as err: + logging.error(err) + return False + logging.error("conn is invalid param") + return False + @classmethod def check_tables_in_db(cls, db_path: any, *tables: any) -> bool: if check_db_path_valid(db_path): @@ -138,46 +175,11 @@ class DBManager: curs.execute(sql) res = curs.fetchone()[0] except sqlite3.Error as err: - print("[ERROR] {}".format(err)) + logging.error(err) finally: cls.destroy_db_connect(conn, curs) return res - @staticmethod - def execute_sql(conn: any, sql: str, params: any = None) -> bool: - """ - execute sql - """ - try: - if isinstance(conn, sqlite3.Connection): - if params: - conn.cursor().execute(sql, params) - else: - conn.cursor().execute(sql) - conn.commit() - return True - except sqlite3.Error as err: - print(f"[ERROR] {err}") - return False - print("[ERROR] conn is invalid param") - return False - - @staticmethod - def executemany_sql(conn: any, sql: str, params: any) -> bool: - """ - execute many sql once - """ - try: - if isinstance(conn, sqlite3.Connection): - conn.cursor().executemany(sql, params) - conn.commit() - return True - except sqlite3.Error as err: - print(f"[ERROR] {err}") - return False - print("[ERROR] conn is invalid param") - return False - @classmethod def fetch_all_data(cls: any, curs: any, sql: str, param: tuple = None, is_dict: bool = True) -> list: """ @@ -192,7 +194,7 @@ class DBManager: else: res = curs.execute(sql) except sqlite3.Error as err: - print(f"[ERROR] {err}") + logging.error(err) curs.row_factory = None return [] try: @@ -204,12 +206,12 @@ class DBManager: else: data += res if len(data) > cls.MAX_ROW_COUNT: - print("[WARRING] The records count in the table exceeds the limit!") + logging.warning("The records count in the table exceeds the limit!") if len(res) < cls.FETCH_SIZE: break return data except sqlite3.Error as err: - print(f"[ERROR] {err}") + logging.error(err) return [] finally: curs.row_factory = None @@ -224,3 +226,4 @@ class CustomizedDictFactory: data_dict = dict(zip(description_set, data)) res.append(data_dict) return res + \ No newline at end of file diff --git a/profiler/cluster_analyse/common_func/empty_class.py b/profiler/cluster_analyse/common_func/empty_class.py index df100d156fa064cca4514260db0b2e843e217d09..44d3529ee18b4beafefe901f2c4b7c5707007ccc 100644 --- a/profiler/cluster_analyse/common_func/empty_class.py +++ b/profiler/cluster_analyse/common_func/empty_class.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + class EmptyClass: def __init__(self: any, info: str = "") -> None: diff --git a/profiler/cluster_analyse/common_func/file_manager.py b/profiler/cluster_analyse/common_func/file_manager.py index 121c4e2d31d9905670445d5aef1de501c099e201..34cfd62fa158476c29411ed5f28c9a69ce541579 100644 --- a/profiler/cluster_analyse/common_func/file_manager.py +++ b/profiler/cluster_analyse/common_func/file_manager.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os import csv import json @@ -22,6 +23,8 @@ import yaml from common_func.constant import Constant from common_func.path_manager import PathManager +logger = logging.getLogger() + class FileManager: DATA_FILE_AUTHORITY = 0o640 @@ -142,9 +145,9 @@ class FileManager: def check_db_path_valid(path: str, is_create: bool = False, max_size: int = Constant.MAX_READ_DB_FILE_BYTES) -> bool: if os.path.islink(path): - print(f'[ERROR] The db file path: {path} is link. Please check the path') + logger.error('The db file path: %s is a link. Please check the path', path) return False if not is_create and os.path.exists(path) and os.path.getsize(path) > max_size: - print(f'[ERROR] The db file: {path} is too large to read. Please check the file') + logger.error('The db file: %s is too large to read. Please check the file', path) return False return True diff --git a/profiler/cluster_analyse/common_func/table_constant.py b/profiler/cluster_analyse/common_func/table_constant.py index de6d47e97e5683493905de5353a9978195e87b70..59c197c06e2fa1185f15742ae8f261b65f4236a2 100644 --- a/profiler/cluster_analyse/common_func/table_constant.py +++ b/profiler/cluster_analyse/common_func/table_constant.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + class TableConstant: RANK_SET = "rank_set" diff --git a/profiler/cluster_analyse/common_func/tables_config.py b/profiler/cluster_analyse/common_func/tables_config.py index d8f49916accfb41a93a1966bf60dc0fdcc5423ef..47c479f11feb759b7c5cc03e7165023b81b0b415 100644 --- a/profiler/cluster_analyse/common_func/tables_config.py +++ b/profiler/cluster_analyse/common_func/tables_config.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + class TablesConfig: DATA = { "ClusterCommAnalyzerTimeMap": [ diff --git a/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py b/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py index bc5810068b04e04aa935ce252ffd127380dd855e..f3fe4f32e64783dab6d7f5a4de8cf923a45cdec6 100644 --- a/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.constant import Constant from compare_backend.utils.common_func import update_order_id diff --git a/profiler/compare_tools/compare_backend/comparator/base_comparator.py b/profiler/compare_tools/compare_backend/comparator/base_comparator.py index 8012dfae94440b7e17613f432770ec8b63ece431..8610bc16e218cf84bba4eb9c955d27b70b5f99bd 100644 --- a/profiler/compare_tools/compare_backend/comparator/base_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/base_comparator.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import ABC, abstractmethod diff --git a/profiler/compare_tools/compare_backend/comparator/communication_comparator.py b/profiler/compare_tools/compare_backend/comparator/communication_comparator.py index f7580bec89a85b8d23e0ec878eda944d95e69f3f..b7b6786223d2360226e7a6686acbb86c5301da74 100644 --- a/profiler/compare_tools/compare_backend/comparator/communication_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/communication_comparator.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.compare_bean.communication_bean import CommunicationBean from compare_backend.utils.constant import Constant diff --git a/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py b/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py index 13c0f776af60f7f250dc22b084cf251733f4c47d..064305cabe26fd8c08d75dabe2f2bc03195d7b60 100644 --- a/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.constant import Constant from compare_backend.utils.common_func import update_order_id diff --git a/profiler/compare_tools/compare_backend/comparator/module_comparetor.py b/profiler/compare_tools/compare_backend/comparator/module_comparetor.py index 49c50b53c5a1b00bd17b7281d80b61d5011cb59a..16f9b6cc4d1e00065c946c5f90eb2009957156d4 100644 --- a/profiler/compare_tools/compare_backend/comparator/module_comparetor.py +++ b/profiler/compare_tools/compare_backend/comparator/module_comparetor.py @@ -1,3 +1,19 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.common_func import update_order_id from compare_backend.utils.constant import Constant @@ -33,4 +49,4 @@ class ModuleComparator(BaseComparator): index += 1 update_order_id(self._rows) if not any(row[-1] != Constant.NA for row in self._rows): - print(f"[WARNING] If you want to see the operator's call stack, you must enable with_stack switch.") + logging.warning(f"If you want to see the operator's call stack, you must enable with_stack switch.") diff --git a/profiler/compare_tools/compare_backend/comparator/module_statistic_comparator.py b/profiler/compare_tools/compare_backend/comparator/module_statistic_comparator.py index e09108f3cbe3744068daf6c5316dc318aea53177..a4e4943601fdbb6da19199e94e50e548b2603781 100644 --- a/profiler/compare_tools/compare_backend/comparator/module_statistic_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/module_statistic_comparator.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from collections import OrderedDict from compare_backend.comparator.base_comparator import BaseComparator diff --git a/profiler/compare_tools/compare_backend/comparator/operator_comparator.py b/profiler/compare_tools/compare_backend/comparator/operator_comparator.py index cc475116cab59104a049689292f25f339a7285ce..00c203cbb73808d84dd689eb2cbe6306256d08c6 100644 --- a/profiler/compare_tools/compare_backend/comparator/operator_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/operator_comparator.py @@ -1,13 +1,34 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from compare_backend.comparator.base_comparator import BaseComparator class OperatorComparator(BaseComparator): def __init__(self, origin_data: any, bean: any): super().__init__(origin_data, bean) + self._initialize_attributes() + + def _initialize_attributes(self): + if self._origin_data is None: + self._origin_data = [] + self._rows = [None] * len(self._origin_data) def _compare(self): if not self._origin_data: return - self._rows = [None] * (len(self._origin_data)) for index, (base_op, comparison_op) in enumerate(self._origin_data): self._rows[index] = self._bean(index, base_op, comparison_op).row + \ No newline at end of file diff --git a/profiler/compare_tools/compare_backend/comparator/operator_statistic_comparator.py b/profiler/compare_tools/compare_backend/comparator/operator_statistic_comparator.py index 73aecf6f1283242311bcb0e848bd94f0f1afa377..9f038c6a6398518973075c1c38f2254d151512ad 100644 --- a/profiler/compare_tools/compare_backend/comparator/operator_statistic_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/operator_statistic_comparator.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.common_func import update_order_id diff --git a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py index 09d8688cf231ba713a2f731c25e1da7d54aa5ddb..a088b724b6fa8a9a5285479012020891333bae20 100644 --- a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.constant import Constant @@ -5,6 +20,8 @@ from compare_backend.utils.constant import Constant class OverallPerformanceComparator(BaseComparator): def __init__(self, origin_data: dict, bean: any): super().__init__(origin_data, bean) + self._headers = [''] + self._rows = [] def _compare(self): base_profiling_info = self._origin_data.get(Constant.BASE_DATA) diff --git a/profiler/compare_tools/compare_backend/interface/compare_interface.py b/profiler/compare_tools/compare_backend/interface/compare_interface.py index 67c5db67fb45c46ba36675931629b8958e6b7bf1..a69135222d06cef8aea89a5d7d57d5896e7134d6 100644 --- a/profiler/compare_tools/compare_backend/interface/compare_interface.py +++ b/profiler/compare_tools/compare_backend/interface/compare_interface.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from compare_backend.utils.constant import Constant from compare_backend.comparator.operator_comparator import OperatorComparator @@ -10,6 +25,7 @@ from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepa from compare_backend.utils.constant import Constant from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching + class CompareInterface: def __init__(self, data_dict: dict, args_manager: any): self._data_dict = data_dict @@ -19,7 +35,8 @@ class CompareInterface: if self._args_manager.enable_kernel_compare: kernel_compare_result = { Constant.BASE_DATA: self._data_dict.get(Constant.BASE_DATA).kernel_details, - Constant.COMPARISON_DATA: self._data_dict.get(Constant.COMPARISON_DATA).kernel_details} + Constant.COMPARISON_DATA: self._data_dict.get(Constant.COMPARISON_DATA).kernel_details, + } return KernelCompareComparator(kernel_compare_result, KernelCompareBean).generate_data() base_op_prepare = OperatorDataPrepare(self._data_dict.get(Constant.BASE_DATA), @@ -30,7 +47,8 @@ class CompareInterface: if self._args_manager.enable_api_compare: api_compare_result = { Constant.BASE_DATA: base_op_prepare.get_all_layer_ops(), - Constant.COMPARISON_DATA: comparison_op_prepare.get_all_layer_ops()} + Constant.COMPARISON_DATA: comparison_op_prepare.get_all_layer_ops(), + } return ApiCompareComparator(api_compare_result, ApiCompareBean).generate_data() if self._args_manager.enable_operator_compare: @@ -42,6 +60,6 @@ class CompareInterface: def _operator_match(self, base_ops, comparison_ops): base_bwd_tid = self._data_dict.get(Constant.BASE_DATA).bwd_tid comparison_bwd_tid = self._data_dict.get(Constant.COMPARISON_DATA).bwd_tid - return SequencePreMatching(self._args_manager.args, base_bwd_tid, comparison_bwd_tid).run(SequencePreMatching.OP_TYPE, - base_ops, comparison_ops) + return (SequencePreMatching(self._args_manager.args, base_bwd_tid, comparison_bwd_tid) + .run(SequencePreMatching.OP_TYPE,base_ops, comparison_ops)) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 2e5df19827d49a3765f64dac60d14a1a5b28c260..b89cc8192406a9df02c2d784203ca2eae1e84dbf 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import abstractmethod, ABC from decimal import Decimal import logging @@ -335,4 +350,4 @@ class BaseProfilingParser(ABC): try: self._trace_events = FileReader.read_trace_file(self._json_path) except Exception: - print(f"[ERROR] Failed to read the file: {self._json_path}") + logger.error("Failed to read the file: %s", self._json_path) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index c2e5a148056ed9a598b1d0303c057927202b5939..f04a904829b4d3aff1724617d464291e439ee33e 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -1,4 +1,20 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys +import logging from collections import defaultdict, Counter from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean @@ -38,9 +54,11 @@ class GPUProfilingParser(BaseProfilingParser): if allocate_bytes > 0: if record: self._result_data.update_memory_list(record) - addr_dict[memory_event.addr] = {Constant.SIZE: allocate_bytes, - Constant.TS: memory_event.start_time, - Constant.ALLOCATION_TIME: memory_event.start_time} + addr_dict[memory_event.addr] = { + Constant.SIZE: allocate_bytes, + Constant.TS: memory_event.start_time, + Constant.ALLOCATION_TIME: memory_event.start_time, + } if allocate_bytes < 0 and record: if abs(allocate_bytes) == record.get(Constant.SIZE): record[Constant.RELEASE_TIME] = memory_event.start_time @@ -127,7 +145,7 @@ class GPUProfilingParser(BaseProfilingParser): def __parse_memory_reserved(self): if not self._memory_events: - print("[INFO] Gpu profiling data doesn't contain memory info.") + logging.info("Gpu profiling data doesn't contain memory info.") return memory_used = max([event.total_reserved for event in self._memory_events]) / 1024 ** 3 self._result_data.overall_metrics.set_memory_used(memory_used) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 8ffb2dadfd0c804beb5a96a65d6f796eba834b5d..047e043f5dac53d8af1176ff0c21f89be5b8cb96 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import sys import logging @@ -240,7 +255,8 @@ class NPUProfilingParser(BaseProfilingParser): uncovered_communication_events, communication_op_events) group_comm_time_dict[self._hccl_tid_name_dict.get(comm_tid)] = { Constant.WAIT_TIME: wait_time, - Constant.TRANSMIT_TIME: uncovered_communication_time - wait_time} + Constant.TRANSMIT_TIME: uncovered_communication_time - wait_time, + } self._result_data.overall_metrics.update_communication_group_time(group_comm_time_dict) def _picking_hccl_event(self, event: TraceEventBean): diff --git a/profiler/compare_tools/compare_backend/view/base_view.py b/profiler/compare_tools/compare_backend/view/base_view.py index d18980b7de2098b5a1015d14fbd1b5be91a23bfc..2c8749bcb5893c22f34a9a2ec184a013427345ba 100644 --- a/profiler/compare_tools/compare_backend/view/base_view.py +++ b/profiler/compare_tools/compare_backend/view/base_view.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import ABC, abstractmethod diff --git a/profiler/compare_tools/compare_backend/view/excel_view.py b/profiler/compare_tools/compare_backend/view/excel_view.py index 73b82b1cd31d7e8207e34a040e484f6387fb8694..2f41a89676e2b5fbbd08a5c6fbf2210528de2aad 100644 --- a/profiler/compare_tools/compare_backend/view/excel_view.py +++ b/profiler/compare_tools/compare_backend/view/excel_view.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os from xlsxwriter import Workbook diff --git a/profiler/compare_tools/compare_backend/view/screen_view.py b/profiler/compare_tools/compare_backend/view/screen_view.py index 150b36c6feda79cafacd7e4980624cd51e116912..d3a6044767635aa891f074e3a0b5b6c3e15a330a 100644 --- a/profiler/compare_tools/compare_backend/view/screen_view.py +++ b/profiler/compare_tools/compare_backend/view/screen_view.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from prettytable import PrettyTable from compare_backend.view.base_view import BaseView diff --git a/profiler/compare_tools/compare_backend/view/work_sheet_creator.py b/profiler/compare_tools/compare_backend/view/work_sheet_creator.py index 58bad621b03f855933517ef9286047e23b5681ea..3dde924d773fd2c30aca6d493410e97431b53255 100644 --- a/profiler/compare_tools/compare_backend/view/work_sheet_creator.py +++ b/profiler/compare_tools/compare_backend/view/work_sheet_creator.py @@ -1,3 +1,18 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from xlsxwriter import Workbook from compare_backend.utils.excel_config import ExcelConfig, CellFormatType @@ -33,8 +48,10 @@ class WorkSheetCreator: if overhead: base_path = f"Base Profiling: {self._args.base_profiling_path}" self._work_sheet.merge_range(overhead[0], base_path, base_header_format) - com_index_range = [self._col_ids.index(overhead[1].split(":")[0][0]), - self._col_ids.index(overhead[1].split(":")[1][0])] + com_index_range = [ + self._col_ids.index(overhead[1].split(":")[0][0]), + self._col_ids.index(overhead[1].split(":")[1][0]), + ] comparison_path = f"Comparison Profiling: {self._args.comparison_profiling_path}" self._work_sheet.merge_range(overhead[1], comparison_path, com_header_format) self._row_id += 2 @@ -47,8 +64,10 @@ class WorkSheetCreator: self._work_sheet.set_column(f"{col_id}:{col_id}", header.get("width")) self._work_sheet.write(f"{col_id}{self._row_id}", header.get("name"), header_format) self._field_format[index] = header.get("type") - ratio_white_list = [ExcelConfig.DIFF_RATIO, ExcelConfig.DIFF_TOTAL_RATIO, - ExcelConfig.DIFF_AVG_RATIO, ExcelConfig.DIFF_CALLS_RATIO, ExcelConfig.DIFF_SELF_RATIO] + ratio_white_list = [ + ExcelConfig.DIFF_RATIO, ExcelConfig.DIFF_TOTAL_RATIO, + ExcelConfig.DIFF_AVG_RATIO, ExcelConfig.DIFF_CALLS_RATIO, ExcelConfig.DIFF_SELF_RATIO, + ] if header.get("name") in ratio_white_list: self._diff_ratio_index.append(index) self._row_id += 1 diff --git a/profiler/compare_tools/compare_interface/comparison_interface.py b/profiler/compare_tools/compare_interface/comparison_interface.py index 936e5a7e8e71838cd5526718ea0113d0a6ff6475..6143e5ee1aa36a038bceb85efda4ba7e9321c7a1 100644 --- a/profiler/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/compare_tools/compare_interface/comparison_interface.py @@ -1,5 +1,21 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys import os +import logging sys.path.append( os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "cluster_analyse")) @@ -26,6 +42,6 @@ class ComparisonInterface: def disaggregate_perf(self, compare_type: str) -> dict: if compare_type != Constant.OVERALL_COMPARE: - print('[ERROR] Invalid compare_type value: {compare_type} which not supported.') + logging.error('Invalid compare_type value: %s which not supported.', compare_type) return {} return OverallPerfInterface(self.base_profiling_path).run() diff --git a/profiler/compare_tools/performance_compare.py b/profiler/compare_tools/performance_compare.py index 419e2c2aff1728c167e760cebc2a6aa7973c34bf..1d71e5e836bbbc4fb1276ccbac0a2ba9b715aece 100644 --- a/profiler/compare_tools/performance_compare.py +++ b/profiler/compare_tools/performance_compare.py @@ -1,14 +1,31 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse import ast import datetime import os.path import sys +import logging sys.path.append( os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cluster_analyse")) from compare_backend.comparison_generator import ComparisonGenerator +logger = logging.getLogger() def main(): parser = argparse.ArgumentParser(description="Compare trace of GPU and NPU") @@ -37,4 +54,4 @@ if __name__ == "__main__": start_time = datetime.datetime.now() main() end_time = datetime.datetime.now() - print(f'[INFO] The comparison task has been completed in a total time of {end_time - start_time}') + logger.info('The comparison task has been completed in a total time of %s seconds.', (end_time - start_time).total_seconds()) \ No newline at end of file