diff --git a/profiler/__init__.py b/profiler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py index 1b4b0c1be4ba466588469728e5aaa0cc3a1422ac..4e31882451950d73df83d73965ed793b257124e9 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py @@ -14,7 +14,7 @@ # limitations under the License. import os -from common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager class AdviceFactory: diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py index 41a648f6e56956f6220132902e6234979f047748..9a63f8e860f5d34e127586ceb006524063970b81 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py @@ -15,7 +15,7 @@ import pandas as pd -from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py index 643eed05c733709e385076a4ebcbef0e510d8ee3..5f2f123fb6867b6b8fc48e8049f6687c5c9369d9 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py @@ -19,7 +19,7 @@ import logging import pandas as pd -from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.advisor.advisor_backend.compute_advice.compute_advice_base import ComputeAdviceBase from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused.op_perf import OpPerfFactory from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant, PerfColor diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py index bb0d73041b6ded5564a9510c17abd594841eaf23..cb280978c41a639a2f5d17e2a0ff08ed3a9962d6 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py @@ -16,9 +16,9 @@ import json import os from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value -from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py index 5ce5b1387a2c7e3fad2c4270526d93e46105c6bb..3a538509b88e7ce3996aa539e49bf714bb163766 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py @@ -17,7 +17,7 @@ import os from collections import defaultdict from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py index 2a9e01fc1371b75b9b5717a266fd3b22a4d90b6b..47846522a9543511b3e55579d05b814d1ca9717d 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py @@ -18,7 +18,7 @@ from collections import defaultdict from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value from msprof_analyze.cluster_analyse.prof_bean.communication_bandwidth_bean import CommunicationBandwidthBean from msprof_analyze.cluster_analyse.prof_bean.communication_time_bean import CommunicationTimeBean diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py index dc43e313c5baab8f57db7fca03b2130c33a73724..e8821b3c2971d1b9707a8347b164f031a44d2922 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py @@ -16,9 +16,9 @@ import os from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value -from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py index a7cffdd176e6ad0c51a9922a9853e1b20f260322..5168f63aef5113221766252ac6ef35f8d9504147 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py @@ -14,7 +14,7 @@ # limitations under the License. import os -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value from msprof_analyze.cluster_analyse.cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean diff --git a/profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py b/profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py deleted file mode 100644 index 9cdaba6cca4bf9e0ae8579d935c7c62211d19a30..0000000000000000000000000000000000000000 --- a/profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -import shutil -import platform -from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager - - - -class PathManager: - MAX_PATH_LENGTH = 4096 - MAX_FILE_NAME_LENGTH = 255 - DATA_FILE_AUTHORITY = 0o640 - DATA_DIR_AUTHORITY = 0o750 - WINDOWS = "windows" - - @classmethod - def check_input_directory_path(cls, path: str): - """ - Function Description: - check whether the path is valid, some businesses can accept a path that does not exist, - so the function do not verify whether the path exists - Parameter: - path: the path to check, whether the incoming path is absolute or relative depends on the business - Exception Description: - when invalid data throw exception - """ - cls.input_path_common_check(path) - base_name = os.path.basename(path) - if os.path.isfile(path): - msg = f"Invalid input path which is a file path: {base_name}" - raise RuntimeError(msg) - - @classmethod - def check_input_file_path(cls, path: str): - """ - Function Description: - check whether the file path is valid, some businesses can accept a path that does not exist, - so the function do not verify whether the path exists - Parameter: - path: the file path to check, whether the incoming path is absolute or relative depends on the business - Exception Description: - when invalid data throw exception - """ - cls.input_path_common_check(path) - base_name = os.path.basename(path) - if os.path.isdir(path): - msg = f"Invalid input path which is a directory path: {base_name}" - raise RuntimeError(msg) - - @classmethod - def check_path_length(cls, path: str): - if len(path) > cls.MAX_PATH_LENGTH: - raise RuntimeError("Length of input path exceeds the limit.") - path_split_list = path.split("/") - for path in path_split_list: - path_list = path.split("\\") - for name in path_list: - if len(name) > cls.MAX_FILE_NAME_LENGTH: - raise RuntimeError("Length of input path exceeds the limit.") - - @classmethod - def input_path_common_check(cls, path: str): - if len(path) > cls.MAX_PATH_LENGTH: - raise RuntimeError("Length of input path exceeds the limit.") - - if os.path.islink(path): - msg = f"Invalid input path which is a soft link." - raise RuntimeError(msg) - - pattern = r'(\.|:|\\|/|_|-|\s|[~0-9a-zA-Z\u4e00-\u9fa5])+' - if not re.fullmatch(pattern, path): - illegal_pattern = r'([^\.\:\\\/\_\-\s~0-9a-zA-Z\u4e00-\u9fa5])+' - invalid_obj = re.search(illegal_pattern, path).group() - msg = f"Invalid path which has illagal characters \"{invalid_obj}\"." - raise RuntimeError(msg) - - path_split_list = path.split("/") - for path in path_split_list: - path_list = path.split("\\") - for name in path_list: - if len(name) > cls.MAX_FILE_NAME_LENGTH: - raise RuntimeError("Length of input path exceeds the limit.") - - @classmethod - def check_path_owner_consistent(cls, path_list: list): - """ - Function Description: - check whether the path belong to process owner - Parameter: - path: the path to check - Exception Description: - when invalid path, prompt the user - """ - if AdditionalArgsManager().force: - return - if platform.system().lower() == cls.WINDOWS: - return - for path in path_list: - if not os.path.exists(path): - continue - if os.stat(path).st_uid != os.getuid(): - check_msg = input("The path does not belong to you, do you want to continue? [y/n]") - if check_msg.lower() != "y": - raise RuntimeError("The user choose not to continue.") - return - - @classmethod - def check_path_writeable(cls, path): - """ - Function Description: - check whether the path is writable - Parameter: - path: the path to check - Exception Description: - when invalid data throw exception - """ - if os.path.islink(path): - msg = f"Invalid path which is a soft link." - raise RuntimeError(msg) - base_name = os.path.basename(path) - if not os.access(path, os.W_OK): - msg = f"The path permission check failed: {base_name}" - raise RuntimeError(msg) - - @classmethod - def check_path_readable(cls, path): - """ - Function Description: - check whether the path is writable - Parameter: - path: the path to check - Exception Description: - when invalid data throw exception - """ - if os.path.islink(path): - msg = f"Invalid path which is a soft link." - raise RuntimeError(msg) - base_name = os.path.basename(path) - if not os.access(path, os.R_OK): - msg = f"The path permission check failed: {base_name}" - raise RuntimeError(msg) - - @classmethod - def remove_path_safety(cls, path: str): - if not os.path.exists(path): - return - base_name = os.path.basename(path) - msg = f"Failed to remove path: {base_name}" - cls.check_path_writeable(path) - if os.path.islink(path): - raise RuntimeError(msg) - try: - shutil.rmtree(path) - except Exception as err: - raise RuntimeError(msg) from err - - @classmethod - def make_dir_safety(cls, path: str): - base_name = os.path.basename(path) - msg = f"Failed to make directory: {base_name}" - if os.path.islink(path): - raise RuntimeError(msg) - if os.path.exists(path): - return - try: - os.makedirs(path, mode=cls.DATA_DIR_AUTHORITY) - except Exception as err: - raise RuntimeError(msg) from err - - @classmethod - def create_file_safety(cls, path: str): - base_name = os.path.basename(path) - msg = f"Failed to create file: {base_name}" - if os.path.islink(path): - raise RuntimeError(msg) - if os.path.exists(path): - return - try: - os.close(os.open(path, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY)) - except Exception as err: - raise RuntimeError(msg) from err - - @classmethod - def get_realpath(cls, path: str) -> str: - if os.path.islink(path): - msg = f"Invalid input path which is a soft link." - raise RuntimeError(msg) - return os.path.abspath(path) diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py index 55f0b7802c61e02ad7900083102fd74ff0a0d411..7d1b4ec250ba1d25079a86f1b0bf95fd2c8906aa 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py @@ -14,7 +14,7 @@ # limitations under the License. import os -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.communication_group.base_communication_group import BaseCommunicationGroup from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/base_recipe_analysis.py b/profiler/msprof_analyze/cluster_analyse/recipes/base_recipe_analysis.py index a2b8de998c906db277e771eb51f50b088607df23..aa9828abbe44509c5b4ad89ae5e576fabf2f8c38 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/base_recipe_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/base_recipe_analysis.py @@ -21,7 +21,7 @@ from abc import abstractmethod, ABC import pandas as pd -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import convert_unit from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py b/profiler/msprof_analyze/prof_common/db_manager.py similarity index 98% rename from profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py rename to profiler/msprof_analyze/prof_common/db_manager.py index fb78057064426869cde23e1ff4e0755e709f9ee9..5d2ffe667fae01d7170d0ad11e9a2f80a288ff74 100644 --- a/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py +++ b/profiler/msprof_analyze/prof_common/db_manager.py @@ -123,9 +123,9 @@ class DBManager: conn.commit() return True except sqlite3.Error as err: - PrintUtils.print_error(f"{err}") + logger.error(err) return False - PrintUtils.print_error("conn is invalid param") + logger.error("conn is invalid param") return False @staticmethod @@ -139,9 +139,9 @@ class DBManager: conn.commit() return True except sqlite3.Error as err: - PrintUtils.print_error(f"{err}") + logger.error(err) return False - PrintUtils.print_error("conn is invalid param") + logger.error("conn is invalid param") return False @staticmethod diff --git a/profiler/msprof_analyze/prof_common/utils.py b/profiler/msprof_analyze/prof_common/utils.py index c4f61f055160e8de07dda9b0659715700372a59a..005d8505c9ccd750d4856518961c62b4407eea1e 100644 --- a/profiler/msprof_analyze/prof_common/utils.py +++ b/profiler/msprof_analyze/prof_common/utils.py @@ -91,17 +91,3 @@ def convert_to_int(num): except (ValueError, NameError): logger.error(f"Can not convert %s to int", num) return 0 - - -class PrintUtils: - @staticmethod - def print_warning(msg): - print("[WARNING] " + msg) - - @staticmethod - def print_info(msg): - print("[INFO] " + msg) - - @staticmethod - def print_error(msg): - print("[ERROR] " + msg) \ No newline at end of file diff --git a/profiler/msprof_analyze/prof_exports/base_stats_export.py b/profiler/msprof_analyze/prof_exports/base_stats_export.py index 4c7cc0a4e72c3b13ff9f8ab7109a2cecfea22cfe..fe86b1ed93ef27a3ee2747a804f832fc0a3f5c53 100644 --- a/profiler/msprof_analyze/prof_exports/base_stats_export.py +++ b/profiler/msprof_analyze/prof_exports/base_stats_export.py @@ -15,7 +15,7 @@ import pandas as pd -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger