From 910a8e8addf325b8afafa772a1b70fff94d8c3bd Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Fri, 7 Feb 2025 17:58:42 +0800 Subject: [PATCH] fix_prof_common_func --- profiler/__init__.py | 0 .../advice_factory/advice_factory.py | 2 +- .../compute_advice/npu_fused/csv_analyzer.py | 2 +- .../compute_advice/npu_slow_advice.py | 2 +- .../analysis/cluster_base_info_analysis.py | 4 +- .../analysis/comm_matrix_analysis.py | 2 +- .../analysis/communication_analysis.py | 2 +- .../analysis/host_info_analysis.py | 4 +- .../analysis/step_trace_time_analysis.py | 2 +- .../common_func/path_manager.py | 203 ------------------ .../communication_db_group.py | 2 +- .../recipes/base_recipe_analysis.py | 2 +- .../common_func => prof_common}/db_manager.py | 8 +- profiler/msprof_analyze/prof_common/utils.py | 14 -- .../prof_exports/base_stats_export.py | 2 +- 15 files changed, 17 insertions(+), 234 deletions(-) create mode 100644 profiler/__init__.py delete mode 100644 profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py rename profiler/msprof_analyze/{cluster_analyse/common_func => prof_common}/db_manager.py (98%) diff --git a/profiler/__init__.py b/profiler/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py index 1b4b0c1be4..4e31882451 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/advice_factory.py @@ -14,7 +14,7 @@ # limitations under the License. import os -from common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager class AdviceFactory: diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py index 41a648f6e5..9a63f8e860 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py @@ -15,7 +15,7 @@ import pandas as pd -from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py index 643eed05c7..5f2f123fb6 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py @@ -19,7 +19,7 @@ import logging import pandas as pd -from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.advisor.advisor_backend.compute_advice.compute_advice_base import ComputeAdviceBase from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused.op_perf import OpPerfFactory from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant, PerfColor diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py index bb0d73041b..cb280978c4 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py @@ -16,9 +16,9 @@ import json import os from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value -from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py index 5ce5b1387a..3a538509b8 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py @@ -17,7 +17,7 @@ import os from collections import defaultdict from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py index 2a9e01fc13..47846522a9 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py @@ -18,7 +18,7 @@ from collections import defaultdict from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value from msprof_analyze.cluster_analyse.prof_bean.communication_bandwidth_bean import CommunicationBandwidthBean from msprof_analyze.cluster_analyse.prof_bean.communication_time_bean import CommunicationTimeBean diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py index dc43e313c5..e8821b3c29 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py @@ -16,9 +16,9 @@ import os from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value -from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager +from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py index a7cffdd176..5168f63aef 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py @@ -14,7 +14,7 @@ # limitations under the License. import os -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value from msprof_analyze.cluster_analyse.cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean diff --git a/profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py b/profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py deleted file mode 100644 index 9cdaba6cca..0000000000 --- a/profiler/msprof_analyze/cluster_analyse/common_func/path_manager.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import re -import shutil -import platform -from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager - - - -class PathManager: - MAX_PATH_LENGTH = 4096 - MAX_FILE_NAME_LENGTH = 255 - DATA_FILE_AUTHORITY = 0o640 - DATA_DIR_AUTHORITY = 0o750 - WINDOWS = "windows" - - @classmethod - def check_input_directory_path(cls, path: str): - """ - Function Description: - check whether the path is valid, some businesses can accept a path that does not exist, - so the function do not verify whether the path exists - Parameter: - path: the path to check, whether the incoming path is absolute or relative depends on the business - Exception Description: - when invalid data throw exception - """ - cls.input_path_common_check(path) - base_name = os.path.basename(path) - if os.path.isfile(path): - msg = f"Invalid input path which is a file path: {base_name}" - raise RuntimeError(msg) - - @classmethod - def check_input_file_path(cls, path: str): - """ - Function Description: - check whether the file path is valid, some businesses can accept a path that does not exist, - so the function do not verify whether the path exists - Parameter: - path: the file path to check, whether the incoming path is absolute or relative depends on the business - Exception Description: - when invalid data throw exception - """ - cls.input_path_common_check(path) - base_name = os.path.basename(path) - if os.path.isdir(path): - msg = f"Invalid input path which is a directory path: {base_name}" - raise RuntimeError(msg) - - @classmethod - def check_path_length(cls, path: str): - if len(path) > cls.MAX_PATH_LENGTH: - raise RuntimeError("Length of input path exceeds the limit.") - path_split_list = path.split("/") - for path in path_split_list: - path_list = path.split("\\") - for name in path_list: - if len(name) > cls.MAX_FILE_NAME_LENGTH: - raise RuntimeError("Length of input path exceeds the limit.") - - @classmethod - def input_path_common_check(cls, path: str): - if len(path) > cls.MAX_PATH_LENGTH: - raise RuntimeError("Length of input path exceeds the limit.") - - if os.path.islink(path): - msg = f"Invalid input path which is a soft link." - raise RuntimeError(msg) - - pattern = r'(\.|:|\\|/|_|-|\s|[~0-9a-zA-Z\u4e00-\u9fa5])+' - if not re.fullmatch(pattern, path): - illegal_pattern = r'([^\.\:\\\/\_\-\s~0-9a-zA-Z\u4e00-\u9fa5])+' - invalid_obj = re.search(illegal_pattern, path).group() - msg = f"Invalid path which has illagal characters \"{invalid_obj}\"." - raise RuntimeError(msg) - - path_split_list = path.split("/") - for path in path_split_list: - path_list = path.split("\\") - for name in path_list: - if len(name) > cls.MAX_FILE_NAME_LENGTH: - raise RuntimeError("Length of input path exceeds the limit.") - - @classmethod - def check_path_owner_consistent(cls, path_list: list): - """ - Function Description: - check whether the path belong to process owner - Parameter: - path: the path to check - Exception Description: - when invalid path, prompt the user - """ - if AdditionalArgsManager().force: - return - if platform.system().lower() == cls.WINDOWS: - return - for path in path_list: - if not os.path.exists(path): - continue - if os.stat(path).st_uid != os.getuid(): - check_msg = input("The path does not belong to you, do you want to continue? [y/n]") - if check_msg.lower() != "y": - raise RuntimeError("The user choose not to continue.") - return - - @classmethod - def check_path_writeable(cls, path): - """ - Function Description: - check whether the path is writable - Parameter: - path: the path to check - Exception Description: - when invalid data throw exception - """ - if os.path.islink(path): - msg = f"Invalid path which is a soft link." - raise RuntimeError(msg) - base_name = os.path.basename(path) - if not os.access(path, os.W_OK): - msg = f"The path permission check failed: {base_name}" - raise RuntimeError(msg) - - @classmethod - def check_path_readable(cls, path): - """ - Function Description: - check whether the path is writable - Parameter: - path: the path to check - Exception Description: - when invalid data throw exception - """ - if os.path.islink(path): - msg = f"Invalid path which is a soft link." - raise RuntimeError(msg) - base_name = os.path.basename(path) - if not os.access(path, os.R_OK): - msg = f"The path permission check failed: {base_name}" - raise RuntimeError(msg) - - @classmethod - def remove_path_safety(cls, path: str): - if not os.path.exists(path): - return - base_name = os.path.basename(path) - msg = f"Failed to remove path: {base_name}" - cls.check_path_writeable(path) - if os.path.islink(path): - raise RuntimeError(msg) - try: - shutil.rmtree(path) - except Exception as err: - raise RuntimeError(msg) from err - - @classmethod - def make_dir_safety(cls, path: str): - base_name = os.path.basename(path) - msg = f"Failed to make directory: {base_name}" - if os.path.islink(path): - raise RuntimeError(msg) - if os.path.exists(path): - return - try: - os.makedirs(path, mode=cls.DATA_DIR_AUTHORITY) - except Exception as err: - raise RuntimeError(msg) from err - - @classmethod - def create_file_safety(cls, path: str): - base_name = os.path.basename(path) - msg = f"Failed to create file: {base_name}" - if os.path.islink(path): - raise RuntimeError(msg) - if os.path.exists(path): - return - try: - os.close(os.open(path, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY)) - except Exception as err: - raise RuntimeError(msg) from err - - @classmethod - def get_realpath(cls, path: str) -> str: - if os.path.islink(path): - msg = f"Invalid input path which is a soft link." - raise RuntimeError(msg) - return os.path.abspath(path) diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py index 55f0b7802c..7d1b4ec250 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py @@ -14,7 +14,7 @@ # limitations under the License. import os -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.communication_group.base_communication_group import BaseCommunicationGroup from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/base_recipe_analysis.py b/profiler/msprof_analyze/cluster_analyse/recipes/base_recipe_analysis.py index a2b8de998c..aa9828abbe 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/base_recipe_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/base_recipe_analysis.py @@ -21,7 +21,7 @@ from abc import abstractmethod, ABC import pandas as pd -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import convert_unit from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py b/profiler/msprof_analyze/prof_common/db_manager.py similarity index 98% rename from profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py rename to profiler/msprof_analyze/prof_common/db_manager.py index fb78057064..5d2ffe667f 100644 --- a/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py +++ b/profiler/msprof_analyze/prof_common/db_manager.py @@ -123,9 +123,9 @@ class DBManager: conn.commit() return True except sqlite3.Error as err: - PrintUtils.print_error(f"{err}") + logger.error(err) return False - PrintUtils.print_error("conn is invalid param") + logger.error("conn is invalid param") return False @staticmethod @@ -139,9 +139,9 @@ class DBManager: conn.commit() return True except sqlite3.Error as err: - PrintUtils.print_error(f"{err}") + logger.error(err) return False - PrintUtils.print_error("conn is invalid param") + logger.error("conn is invalid param") return False @staticmethod diff --git a/profiler/msprof_analyze/prof_common/utils.py b/profiler/msprof_analyze/prof_common/utils.py index c4f61f0551..005d8505c9 100644 --- a/profiler/msprof_analyze/prof_common/utils.py +++ b/profiler/msprof_analyze/prof_common/utils.py @@ -91,17 +91,3 @@ def convert_to_int(num): except (ValueError, NameError): logger.error(f"Can not convert %s to int", num) return 0 - - -class PrintUtils: - @staticmethod - def print_warning(msg): - print("[WARNING] " + msg) - - @staticmethod - def print_info(msg): - print("[INFO] " + msg) - - @staticmethod - def print_error(msg): - print("[ERROR] " + msg) \ No newline at end of file diff --git a/profiler/msprof_analyze/prof_exports/base_stats_export.py b/profiler/msprof_analyze/prof_exports/base_stats_export.py index 4c7cc0a4e7..fe86b1ed93 100644 --- a/profiler/msprof_analyze/prof_exports/base_stats_export.py +++ b/profiler/msprof_analyze/prof_exports/base_stats_export.py @@ -15,7 +15,7 @@ import pandas as pd -from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger -- Gitee