From f6f1c05ce44373bda7ecd19c0810d660093d72c4 Mon Sep 17 00:00:00 2001 From: swagglian <1312140415@qq.com> Date: Tue, 1 Jul 2025 03:19:29 +0000 Subject: [PATCH 01/10] update ascend_deployer/utils.py. Signed-off-by: swagglian <1312140415@qq.com> --- ascend_deployer/utils.py | 77 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/ascend_deployer/utils.py b/ascend_deployer/utils.py index f6938a53..89cb2088 100644 --- a/ascend_deployer/utils.py +++ b/ascend_deployer/utils.py @@ -17,7 +17,7 @@ import json import shlex import stat -import sys +import string import argparse import getpass import logging @@ -26,6 +26,8 @@ import platform import shutil import re import os +import sys +import errno from subprocess import PIPE, Popen ROOT_PATH = SRC_PATH = os.path.dirname(__file__) @@ -304,3 +306,76 @@ def get_hosts_name(tags): if (isinstance(tags, str) and tags in dl_items) or (isinstance(tags, list) and set(tags) & set(dl_items)): return 'master,worker' return 'worker' + + + +PATH_WHITE_LIST_LIN = string.digits + string.ascii_letters + '~-+_./ ' +MAX_PATH_LEN = 4096 +def get_validated_env( + env_name, + whitelist=PATH_WHITE_LIST_LIN, + min_length=1, + max_length=MAX_PATH_LEN, + check_symlink=True +): + """ + 获取并验证环境变量 (兼容 Python 2/3) + :param env_name: 环境变量名称 + :param whitelist: 允许的值列表 + :param min_length: 最小长度限制 + :param max_length: 最大长度限制 + :param check_symlink: 是否检查软链接 + :return: 验证通过的环境变量值 + :raises ValueError: 验证失败时抛出 + """ + value = os.getenv(env_name) + + if value is None: + return None + + # 白名单校验 + for char in value: + if char not in whitelist: + raise ValueError( + "The path is invalid. The path can contain only char in '{}'".format(whitelist)) + + # 长度校验 + str_len = len(value) + if min_length is not None and str_len < min_length: + raise ValueError( + "Value for {} is too short. Minimum length: {}, actual: {}".format( + env_name, min_length, str_len + ) + ) + + if max_length is not None and str_len > max_length: + raise ValueError( + "Value for {} is too long. Maximum length: {}, actual: {}".format( + env_name, max_length, str_len + ) + ) + + # 路径安全校验 + if check_symlink: + # 在 Python 2/3 中正确处理 unicode 路径 + if isinstance(value, bytes): + path_value = value.decode('utf-8', 'replace') + else: + path_value = value + # 软链接检查 + if check_symlink: + try: + # 检查路径是否存在且是符号链接 + if os.path.lexists(path_value) and os.path.islink(path_value): + raise ValueError( + "Path for {} is a symlink: {}. Symlinks are not allowed for security reasons.".format( + env_name, path_value + ) + ) + except (OSError, IOError) as e: + # 处理文件系统访问错误 + if e.errno != errno.ENOENT: # 忽略文件不存在的错误 + raise ValueError( + "Error checking symlink for {}: {} - {}".format(env_name, path_value, str(e)) + ) + return value \ No newline at end of file -- Gitee From ef59e6185e53cad6a886f6265f6520c2c885c126 Mon Sep 17 00:00:00 2001 From: swagglian <1312140415@qq.com> Date: Tue, 1 Jul 2025 03:20:21 +0000 Subject: [PATCH 02/10] add test/test_utils.py. Signed-off-by: swagglian <1312140415@qq.com> --- test/test_utils.py | 381 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 test/test_utils.py diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 00000000..69f09d10 --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,381 @@ +#!/usr/bin/env python3 +# coding: utf-8 +# Copyright 2023 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =========================================================================== +import json +import shlex +import stat +import string +import argparse +import getpass +import logging +import logging.handlers +import platform +import shutil +import re +import os +import sys +import errno +from subprocess import PIPE, Popen + +ROOT_PATH = SRC_PATH = os.path.dirname(__file__) +NEXUS_SENTINEL_FILE = os.path.expanduser('~/.local/nexus.sentinel') +MODE_700 = stat.S_IRWXU +MODE_600 = stat.S_IRUSR | stat.S_IWUSR +MODE_400 = stat.S_IRUSR + +LOG = logging.getLogger('ascend_deployer.utils') +MAX_LEN = 120 + +dir_list = ['downloader', 'playbooks', 'tools', 'ansible_plugin', 'group_vars', 'patch', 'scripts', 'yamls', + 'library', 'module_utils', 'templates'] +file_list = ['install.sh', 'inventory_file', 'ansible.cfg', + '__init__.py', 'ascend_deployer.py', 'jobs.py', 'utils.py', + 'version.json'] + +VERSION_PATTERN = re.compile(r"(\d+)") + + +def compare_version(src_version, target_version): + use_version_parts = VERSION_PATTERN.split(src_version) + new_version_parts = VERSION_PATTERN.split(target_version) + for cur_ver_part, new_ver_part in zip(use_version_parts, new_version_parts): + if cur_ver_part.isdigit() and new_ver_part.isdigit(): + result = int(cur_ver_part) - int(new_ver_part) + else: + result = (cur_ver_part > new_ver_part) - (cur_ver_part < new_ver_part) + if result != 0: + return result + return len(use_version_parts) - len(new_version_parts) + + +def copy_scripts(): + """ + copy scripts from library to ASCEND_DEPLOY_HOME + the default ASCEND_DEPLOYER_HOME is HOME + """ + if SRC_PATH == ROOT_PATH: + return + + if not os.path.exists(ROOT_PATH): + os.makedirs(ROOT_PATH, mode=0o750) + for dir_name in dir_list: + src = os.path.join(SRC_PATH, dir_name) + dst = os.path.join(ROOT_PATH, dir_name) + if os.path.exists(src) and not os.path.exists(dst): + shutil.copytree(src, dst) + + for filename in file_list: + src = os.path.join(SRC_PATH, filename) + dst = os.path.join(ROOT_PATH, filename) + if not os.path.exists(dst) and os.path.exists(src): + shutil.copy(src, dst) + + +if 'site-packages' in ROOT_PATH or 'dist-packages' in ROOT_PATH: + deployer_home = os.getcwd() + if platform.system() == 'Linux': + deployer_home = os.getenv('ASCEND_DEPLOYER_HOME', os.getenv('HOME')) + ROOT_PATH = os.path.join(deployer_home, 'ascend-deployer') + copy_scripts() + + +class ValidChoices(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, list(set(values))) + + +class SkipCheck(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + if value.lower() == "true": + setattr(namespace, self.dest, True) + return + setattr(namespace, self.dest, False) + + +def pretty_format(text): + results = [] + loc = text.index(':') + 1 + results.append(text[:loc]) + results.extend(text[loc:].split(',')) + return results + + +class HelpFormatter(argparse.HelpFormatter): + def _split_lines(self, text, width): + if ':' in text: + return pretty_format(text) + import textwrap + return textwrap.wrap(text, width, break_on_hyphens=False) + + +def args_with_comma(args): + new_args = [] + for arg in args: + sep_loc = arg.find('=') + ver_loc = arg.find('==') + if sep_loc > 0 and sep_loc != ver_loc: + new_args.append(arg[:sep_loc]) + arg = arg[sep_loc + 1:] + for sub_arg in arg.split(','): + if sub_arg: + new_args.append(sub_arg) + return new_args + + +def get_python_version_list(): + origin_py_version_file = os.path.join(ROOT_PATH, 'downloader', 'python_version.json') + update_py_version = os.path.join(ROOT_PATH, 'downloader', 'obs_downloader_config', 'python_version.json') + python_version_json = update_py_version if os.path.exists(update_py_version) else origin_py_version_file + with open(python_version_json, 'r') as json_file: + data = json.load(json_file) + available_python_list = [item['filename'].rstrip('.tar.xz') for item in data] + return available_python_list + + +def get_name_list(dir_path, prefix, suffix): + items = [] + for file_name in os.listdir(dir_path): + if file_name.startswith(prefix) and file_name.endswith(suffix): + item = file_name.replace(prefix, '').replace(suffix, '') + items.append(item) + return sorted(items) + + +dl_items = ['ascend-device-plugin', 'ascend-docker-runtime', 'ascend-operator', 'hccl-controller', 'mindio', 'noded', + 'npu-exporter', 'resilience-controller', 'volcano', 'clusterd', 'dl'] +install_items = get_name_list(os.path.join(ROOT_PATH, "playbooks", "install"), 'install_', '.yml') +scene_items = get_name_list(os.path.join(ROOT_PATH, "playbooks", "scene"), 'scene_', '.yml') +patch_items = get_name_list(os.path.join(ROOT_PATH, "playbooks", "install", "patch"), "install_", ".yml") +upgrade_items = get_name_list(os.path.join(ROOT_PATH, "playbooks", "install", "upgrade"), "upgrade_", ".yml") +test_items = ['all', 'firmware', 'driver', 'nnrt', 'nnae', 'toolkit', 'toolbox', 'mindspore', 'pytorch', + 'tensorflow', 'tfplugin', 'fault-diag', 'ascend-docker-runtime', 'ascend-device-plugin', 'volcano', + 'noded', 'clusterd', 'hccl-controller', 'ascend-operator', 'npu-exporter', 'resilience-controller', + 'mindie_image', 'mcu'] +check_items = ['full', 'fast'] + +LOG_MAX_BACKUP_COUNT = 5 +LOG_MAX_SIZE = 20 * 1024 * 1024 +LOG_FILE = os.path.join(ROOT_PATH, 'install.log') +LOG_OPERATION_FILE = os.path.join(ROOT_PATH, 'install_operation.log') + + +class UserHostFilter(logging.Filter): + user = getpass.getuser() + host = os.getenv('SSH_CLIENT', 'localhost').split()[0] + + def filter(self, record): + record.user = self.user + record.host = self.host + return True + + +class RotatingFileHandler(logging.handlers.RotatingFileHandler): + def doRollover(self): + try: + os.chmod(self.baseFilename, 0o400) + except OSError: + os.chmod('{}.{}'.format(self.baseFilename, LOG_MAX_BACKUP_COUNT), 0o600) + finally: + logging.handlers.RotatingFileHandler.doRollover(self) + os.chmod(self.baseFilename, 0o600) + + +LOGGING_CONFIG = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + 'extra': { + 'format': "%(asctime)s %(user)s@%(host)s [%(levelname)s] " + "[%(filename)s:%(lineno)d %(funcName)s] %(message)s" + } + }, + "filters": { + "user_host": { + '()': UserHostFilter + } + }, + "handlers": { + "install": { + "level": "DEBUG", + "formatter": "extra", + "class": 'utils.RotatingFileHandler', + "filename": LOG_FILE, + 'maxBytes': LOG_MAX_SIZE, + 'backupCount': LOG_MAX_BACKUP_COUNT, + 'encoding': "UTF-8", + "filters": ["user_host"], + }, + "install_operation": { + "level": "INFO", + "formatter": "extra", + "class": 'utils.RotatingFileHandler', + "filename": LOG_OPERATION_FILE, + 'maxBytes': LOG_MAX_SIZE, + 'backupCount': LOG_MAX_BACKUP_COUNT, + 'encoding': "UTF-8", + "filters": ["user_host"], + }, + }, + "loggers": { + "ascend_deployer": { + "handlers": ["install"], + "level": "INFO", + "propagate": True, + }, + "install_operation": { + "handlers": ["install_operation"], + "level": "INFO", + "propagate": True, + }, + } +} + + +def run_cmd(args, oneline=False, **kwargs): + if not kwargs.get('shell') and isinstance(args, str): + args = shlex.split(args, posix=platform.system() == 'Linux') + cmd = args if isinstance(args, str) else ' '.join(args) + LOG.info(cmd.center(MAX_LEN, '-')) + stdout = kwargs.pop('stdout', PIPE if oneline else None) + stderr = kwargs.pop('stderr', PIPE) + text = kwargs.pop('universal_newlines', True) + output = [] + process = Popen(args, stdout=stdout, stderr=stderr, universal_newlines=text, **kwargs) + if oneline: + for line in iter(process.stdout.readline, ''): + line = line.strip() + output.append(line) + LOG.info(line) + if len(line) <= MAX_LEN: + line += (MAX_LEN - len(line)) * " " + else: + # if the line too long(> MAX_LEN), only print first (MAX_LEN -3) characters and '...' + line = line[0:MAX_LEN - 4] + "..." + sys.stdout.write("\r{}".format(line)) + err = process.stderr.read() + process.wait() + else: + out, err = process.communicate() + if isinstance(out, str): + output = out.splitlines() + for line in output: + LOG.info(line) + if process.returncode: + if err and '[ASCEND][WARNING]' not in str(err): + raise Exception(err) + raise Exception("returned non-zero exit status {}".format(process.returncode)) + elif err and '[ASCEND][WARNING]' in str(err): + print(err) + return output + + +def install_pkg(name, *paths): + from distutils.spawn import find_executable + if find_executable(name): + LOG.info('{} is already installed, skip'.format(name)) + return + if find_executable('dpkg'): + prefix_cmd = "dpkg --force-all -i" + suffix_cmd = '.deb' + else: + prefix_cmd = "rpm -ivUh --force --nodeps --replacepkgs" + suffix_cmd = '.rpm' + pkg_path = os.path.join(ROOT_PATH, 'resources', *paths) + if not pkg_path.endswith(('.deb', '.rpm')): + pkg_path += suffix_cmd + cmd = "{} {}".format(prefix_cmd, pkg_path) + if getpass.getuser() != 'root': + raise Exception('no permission to run cmd: {}, please run command with root user firstly'.format(cmd)) + return run_cmd(cmd, oneline=True, shell=True) + + +def get_hosts_name(tags): + if (isinstance(tags, str) and tags in dl_items) or (isinstance(tags, list) and set(tags) & set(dl_items)): + return 'master,worker' + return 'worker' + + + +PATH_WHITE_LIST_LIN = string.digits + string.ascii_letters + '~-+_./ ' +MAX_PATH_LEN = 4096 +def get_validated_env( + env_name, + whitelist=PATH_WHITE_LIST_LIN, + min_length=1, + max_length=MAX_PATH_LEN, + check_symlink=True +): + """ + 获取并验证环境变量 (兼容 Python 2/3) + :param env_name: 环境变量名称 + :param whitelist: 允许的值列表 + :param min_length: 最小长度限制 + :param max_length: 最大长度限制 + :param check_symlink: 是否检查软链接 + :return: 验证通过的环境变量值 + :raises ValueError: 验证失败时抛出 + """ + value = os.getenv(env_name) + + if value is None: + return None + + # 白名单校验 + for char in value: + if char not in whitelist: + raise ValueError( + "The path is invalid. The path can contain only char in '{}'".format(whitelist)) + + # 长度校验 + str_len = len(value) + if min_length is not None and str_len < min_length: + raise ValueError( + "Value for {} is too short. Minimum length: {}, actual: {}".format( + env_name, min_length, str_len + ) + ) + + if max_length is not None and str_len > max_length: + raise ValueError( + "Value for {} is too long. Maximum length: {}, actual: {}".format( + env_name, max_length, str_len + ) + ) + + # 路径安全校验 + if check_symlink: + # 在 Python 2/3 中正确处理 unicode 路径 + if isinstance(value, bytes): + path_value = value.decode('utf-8', 'replace') + else: + path_value = value + # 软链接检查 + if check_symlink: + try: + # 检查路径是否存在且是符号链接 + if os.path.lexists(path_value) and os.path.islink(path_value): + raise ValueError( + "Path for {} is a symlink: {}. Symlinks are not allowed for security reasons.".format( + env_name, path_value + ) + ) + except (OSError, IOError) as e: + # 处理文件系统访问错误 + if e.errno != errno.ENOENT: # 忽略文件不存在的错误 + raise ValueError( + "Error checking symlink for {}: {} - {}".format(env_name, path_value, str(e)) + ) + return value \ No newline at end of file -- Gitee From be7d8236d761ff468954a66f84a59c8d01d3627f Mon Sep 17 00:00:00 2001 From: swagglian <1312140415@qq.com> Date: Tue, 1 Jul 2025 03:20:36 +0000 Subject: [PATCH 03/10] update test/test_utils.py. Signed-off-by: swagglian <1312140415@qq.com> --- test/test_utils.py | 445 +++++++-------------------------------------- 1 file changed, 69 insertions(+), 376 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index 69f09d10..cf82dcb2 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,381 +1,74 @@ -#!/usr/bin/env python3 -# coding: utf-8 -# Copyright 2023 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =========================================================================== -import json -import shlex -import stat import string -import argparse -import getpass -import logging -import logging.handlers -import platform -import shutil -import re -import os -import sys +import unittest import errno -from subprocess import PIPE, Popen - -ROOT_PATH = SRC_PATH = os.path.dirname(__file__) -NEXUS_SENTINEL_FILE = os.path.expanduser('~/.local/nexus.sentinel') -MODE_700 = stat.S_IRWXU -MODE_600 = stat.S_IRUSR | stat.S_IWUSR -MODE_400 = stat.S_IRUSR - -LOG = logging.getLogger('ascend_deployer.utils') -MAX_LEN = 120 - -dir_list = ['downloader', 'playbooks', 'tools', 'ansible_plugin', 'group_vars', 'patch', 'scripts', 'yamls', - 'library', 'module_utils', 'templates'] -file_list = ['install.sh', 'inventory_file', 'ansible.cfg', - '__init__.py', 'ascend_deployer.py', 'jobs.py', 'utils.py', - 'version.json'] - -VERSION_PATTERN = re.compile(r"(\d+)") - - -def compare_version(src_version, target_version): - use_version_parts = VERSION_PATTERN.split(src_version) - new_version_parts = VERSION_PATTERN.split(target_version) - for cur_ver_part, new_ver_part in zip(use_version_parts, new_version_parts): - if cur_ver_part.isdigit() and new_ver_part.isdigit(): - result = int(cur_ver_part) - int(new_ver_part) - else: - result = (cur_ver_part > new_ver_part) - (cur_ver_part < new_ver_part) - if result != 0: - return result - return len(use_version_parts) - len(new_version_parts) - - -def copy_scripts(): - """ - copy scripts from library to ASCEND_DEPLOY_HOME - the default ASCEND_DEPLOYER_HOME is HOME - """ - if SRC_PATH == ROOT_PATH: - return - - if not os.path.exists(ROOT_PATH): - os.makedirs(ROOT_PATH, mode=0o750) - for dir_name in dir_list: - src = os.path.join(SRC_PATH, dir_name) - dst = os.path.join(ROOT_PATH, dir_name) - if os.path.exists(src) and not os.path.exists(dst): - shutil.copytree(src, dst) - - for filename in file_list: - src = os.path.join(SRC_PATH, filename) - dst = os.path.join(ROOT_PATH, filename) - if not os.path.exists(dst) and os.path.exists(src): - shutil.copy(src, dst) - - -if 'site-packages' in ROOT_PATH or 'dist-packages' in ROOT_PATH: - deployer_home = os.getcwd() - if platform.system() == 'Linux': - deployer_home = os.getenv('ASCEND_DEPLOYER_HOME', os.getenv('HOME')) - ROOT_PATH = os.path.join(deployer_home, 'ascend-deployer') - copy_scripts() - - -class ValidChoices(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, list(set(values))) - - -class SkipCheck(argparse.Action): - def __call__(self, parser, namespace, value, option_string=None): - if value.lower() == "true": - setattr(namespace, self.dest, True) - return - setattr(namespace, self.dest, False) - - -def pretty_format(text): - results = [] - loc = text.index(':') + 1 - results.append(text[:loc]) - results.extend(text[loc:].split(',')) - return results - - -class HelpFormatter(argparse.HelpFormatter): - def _split_lines(self, text, width): - if ':' in text: - return pretty_format(text) - import textwrap - return textwrap.wrap(text, width, break_on_hyphens=False) - - -def args_with_comma(args): - new_args = [] - for arg in args: - sep_loc = arg.find('=') - ver_loc = arg.find('==') - if sep_loc > 0 and sep_loc != ver_loc: - new_args.append(arg[:sep_loc]) - arg = arg[sep_loc + 1:] - for sub_arg in arg.split(','): - if sub_arg: - new_args.append(sub_arg) - return new_args - - -def get_python_version_list(): - origin_py_version_file = os.path.join(ROOT_PATH, 'downloader', 'python_version.json') - update_py_version = os.path.join(ROOT_PATH, 'downloader', 'obs_downloader_config', 'python_version.json') - python_version_json = update_py_version if os.path.exists(update_py_version) else origin_py_version_file - with open(python_version_json, 'r') as json_file: - data = json.load(json_file) - available_python_list = [item['filename'].rstrip('.tar.xz') for item in data] - return available_python_list - - -def get_name_list(dir_path, prefix, suffix): - items = [] - for file_name in os.listdir(dir_path): - if file_name.startswith(prefix) and file_name.endswith(suffix): - item = file_name.replace(prefix, '').replace(suffix, '') - items.append(item) - return sorted(items) - - -dl_items = ['ascend-device-plugin', 'ascend-docker-runtime', 'ascend-operator', 'hccl-controller', 'mindio', 'noded', - 'npu-exporter', 'resilience-controller', 'volcano', 'clusterd', 'dl'] -install_items = get_name_list(os.path.join(ROOT_PATH, "playbooks", "install"), 'install_', '.yml') -scene_items = get_name_list(os.path.join(ROOT_PATH, "playbooks", "scene"), 'scene_', '.yml') -patch_items = get_name_list(os.path.join(ROOT_PATH, "playbooks", "install", "patch"), "install_", ".yml") -upgrade_items = get_name_list(os.path.join(ROOT_PATH, "playbooks", "install", "upgrade"), "upgrade_", ".yml") -test_items = ['all', 'firmware', 'driver', 'nnrt', 'nnae', 'toolkit', 'toolbox', 'mindspore', 'pytorch', - 'tensorflow', 'tfplugin', 'fault-diag', 'ascend-docker-runtime', 'ascend-device-plugin', 'volcano', - 'noded', 'clusterd', 'hccl-controller', 'ascend-operator', 'npu-exporter', 'resilience-controller', - 'mindie_image', 'mcu'] -check_items = ['full', 'fast'] - -LOG_MAX_BACKUP_COUNT = 5 -LOG_MAX_SIZE = 20 * 1024 * 1024 -LOG_FILE = os.path.join(ROOT_PATH, 'install.log') -LOG_OPERATION_FILE = os.path.join(ROOT_PATH, 'install_operation.log') - - -class UserHostFilter(logging.Filter): - user = getpass.getuser() - host = os.getenv('SSH_CLIENT', 'localhost').split()[0] - - def filter(self, record): - record.user = self.user - record.host = self.host - return True - - -class RotatingFileHandler(logging.handlers.RotatingFileHandler): - def doRollover(self): - try: - os.chmod(self.baseFilename, 0o400) - except OSError: - os.chmod('{}.{}'.format(self.baseFilename, LOG_MAX_BACKUP_COUNT), 0o600) - finally: - logging.handlers.RotatingFileHandler.doRollover(self) - os.chmod(self.baseFilename, 0o600) - - -LOGGING_CONFIG = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - 'extra': { - 'format': "%(asctime)s %(user)s@%(host)s [%(levelname)s] " - "[%(filename)s:%(lineno)d %(funcName)s] %(message)s" - } - }, - "filters": { - "user_host": { - '()': UserHostFilter - } - }, - "handlers": { - "install": { - "level": "DEBUG", - "formatter": "extra", - "class": 'utils.RotatingFileHandler', - "filename": LOG_FILE, - 'maxBytes': LOG_MAX_SIZE, - 'backupCount': LOG_MAX_BACKUP_COUNT, - 'encoding': "UTF-8", - "filters": ["user_host"], - }, - "install_operation": { - "level": "INFO", - "formatter": "extra", - "class": 'utils.RotatingFileHandler', - "filename": LOG_OPERATION_FILE, - 'maxBytes': LOG_MAX_SIZE, - 'backupCount': LOG_MAX_BACKUP_COUNT, - 'encoding': "UTF-8", - "filters": ["user_host"], - }, - }, - "loggers": { - "ascend_deployer": { - "handlers": ["install"], - "level": "INFO", - "propagate": True, - }, - "install_operation": { - "handlers": ["install_operation"], - "level": "INFO", - "propagate": True, - }, - } -} - - -def run_cmd(args, oneline=False, **kwargs): - if not kwargs.get('shell') and isinstance(args, str): - args = shlex.split(args, posix=platform.system() == 'Linux') - cmd = args if isinstance(args, str) else ' '.join(args) - LOG.info(cmd.center(MAX_LEN, '-')) - stdout = kwargs.pop('stdout', PIPE if oneline else None) - stderr = kwargs.pop('stderr', PIPE) - text = kwargs.pop('universal_newlines', True) - output = [] - process = Popen(args, stdout=stdout, stderr=stderr, universal_newlines=text, **kwargs) - if oneline: - for line in iter(process.stdout.readline, ''): - line = line.strip() - output.append(line) - LOG.info(line) - if len(line) <= MAX_LEN: - line += (MAX_LEN - len(line)) * " " - else: - # if the line too long(> MAX_LEN), only print first (MAX_LEN -3) characters and '...' - line = line[0:MAX_LEN - 4] + "..." - sys.stdout.write("\r{}".format(line)) - err = process.stderr.read() - process.wait() - else: - out, err = process.communicate() - if isinstance(out, str): - output = out.splitlines() - for line in output: - LOG.info(line) - if process.returncode: - if err and '[ASCEND][WARNING]' not in str(err): - raise Exception(err) - raise Exception("returned non-zero exit status {}".format(process.returncode)) - elif err and '[ASCEND][WARNING]' in str(err): - print(err) - return output - - -def install_pkg(name, *paths): - from distutils.spawn import find_executable - if find_executable(name): - LOG.info('{} is already installed, skip'.format(name)) - return - if find_executable('dpkg'): - prefix_cmd = "dpkg --force-all -i" - suffix_cmd = '.deb' - else: - prefix_cmd = "rpm -ivUh --force --nodeps --replacepkgs" - suffix_cmd = '.rpm' - pkg_path = os.path.join(ROOT_PATH, 'resources', *paths) - if not pkg_path.endswith(('.deb', '.rpm')): - pkg_path += suffix_cmd - cmd = "{} {}".format(prefix_cmd, pkg_path) - if getpass.getuser() != 'root': - raise Exception('no permission to run cmd: {}, please run command with root user firstly'.format(cmd)) - return run_cmd(cmd, oneline=True, shell=True) - - -def get_hosts_name(tags): - if (isinstance(tags, str) and tags in dl_items) or (isinstance(tags, list) and set(tags) & set(dl_items)): - return 'master,worker' - return 'worker' - - - +from unittest.mock import patch +from ascend_deployer.utils import get_validated_env PATH_WHITE_LIST_LIN = string.digits + string.ascii_letters + '~-+_./ ' MAX_PATH_LEN = 4096 -def get_validated_env( - env_name, - whitelist=PATH_WHITE_LIST_LIN, - min_length=1, - max_length=MAX_PATH_LEN, - check_symlink=True -): - """ - 获取并验证环境变量 (兼容 Python 2/3) - :param env_name: 环境变量名称 - :param whitelist: 允许的值列表 - :param min_length: 最小长度限制 - :param max_length: 最大长度限制 - :param check_symlink: 是否检查软链接 - :return: 验证通过的环境变量值 - :raises ValueError: 验证失败时抛出 - """ - value = os.getenv(env_name) - - if value is None: - return None - - # 白名单校验 - for char in value: - if char not in whitelist: - raise ValueError( - "The path is invalid. The path can contain only char in '{}'".format(whitelist)) - - # 长度校验 - str_len = len(value) - if min_length is not None and str_len < min_length: - raise ValueError( - "Value for {} is too short. Minimum length: {}, actual: {}".format( - env_name, min_length, str_len - ) - ) - - if max_length is not None and str_len > max_length: - raise ValueError( - "Value for {} is too long. Maximum length: {}, actual: {}".format( - env_name, max_length, str_len - ) - ) - # 路径安全校验 - if check_symlink: - # 在 Python 2/3 中正确处理 unicode 路径 - if isinstance(value, bytes): - path_value = value.decode('utf-8', 'replace') - else: - path_value = value - # 软链接检查 - if check_symlink: - try: - # 检查路径是否存在且是符号链接 - if os.path.lexists(path_value) and os.path.islink(path_value): - raise ValueError( - "Path for {} is a symlink: {}. Symlinks are not allowed for security reasons.".format( - env_name, path_value - ) - ) - except (OSError, IOError) as e: - # 处理文件系统访问错误 - if e.errno != errno.ENOENT: # 忽略文件不存在的错误 - raise ValueError( - "Error checking symlink for {}: {} - {}".format(env_name, path_value, str(e)) - ) - return value \ No newline at end of file +class TestGetValidatedEnv(unittest.TestCase): + + @patch('os.getenv') + def test_env_not_set(self, mock_getenv): + mock_getenv.return_value = None + with self.assertRaises(ValueError) as context: + get_validated_env('TEST_ENV') + self.assertEqual(str(context.exception), "Environment variable TEST_ENV is not set") + + @patch('os.getenv') + def test_env_value_not_in_whitelist(self, mock_getenv): + mock_getenv.return_value = 'invalid_value(' + with self.assertRaises(ValueError) as context: + get_validated_env('TEST_ENV') + self.assertEqual(str(context.exception), "The path is invalid. The path can contain only char in '{}'".format(PATH_WHITE_LIST_LIN)) + + @patch('os.getenv') + def test_env_value_too_short(self, mock_getenv): + mock_getenv.return_value = 'a' + with self.assertRaises(ValueError) as context: + get_validated_env('TEST_ENV', min_length=2) + self.assertEqual(str(context.exception), "Value for TEST_ENV is too short. Minimum length: 2, actual: 1") + + @patch('os.getenv') + def test_env_value_too_long(self, mock_getenv): + mock_getenv.return_value = 'a' * (MAX_PATH_LEN + 1) + with self.assertRaises(ValueError) as context: + get_validated_env('TEST_ENV', max_length=MAX_PATH_LEN) + self.assertEqual(str(context.exception), "Value for TEST_ENV is too long. Maximum length: {}, actual: {}".format(MAX_PATH_LEN, MAX_PATH_LEN + 1)) + + @patch('os.getenv') + @patch('os.path.lexists') + @patch('os.path.islink') + def test_env_value_is_symlink(self, mock_islink, mock_lexists, mock_getenv): + mock_getenv.return_value = '/path/to/symlink' + mock_lexists.return_value = True + mock_islink.return_value = True + with self.assertRaises(ValueError) as context: + get_validated_env('TEST_ENV', check_symlink=True) + self.assertEqual(str(context.exception), "Path for TEST_ENV is a symlink: /path/to/symlink. Symlinks are not allowed for security reasons.") + + @patch('os.getenv') + @patch('os.path.lexists') + @patch('os.path.islink') + def test_env_value_is_not_symlink(self, mock_islink, mock_lexists, mock_getenv): + mock_getenv.return_value = '/path/to/file' + mock_lexists.return_value = True + mock_islink.return_value = False + self.assertEqual(get_validated_env('TEST_ENV', check_symlink=True), '/path/to/file') + + @patch('os.getenv') + @patch('os.path.lexists') + def test_env_value_lexists_error(self, mock_lexists, mock_getenv): + mock_getenv.return_value = '/path/to/file' + mock_lexists.side_effect = OSError(errno.ENOENT, 'No such file or directory') + self.assertEqual(get_validated_env('TEST_ENV', check_symlink=True), '/path/to/file') + + @patch('os.getenv') + @patch('os.path.lexists') + def test_env_value_lexists_io_error(self, mock_lexists, mock_getenv): + mock_getenv.return_value = '/path/to/file' + mock_lexists.side_effect = IOError(errno.ENOENT, 'No such file or directory') + self.assertEqual(get_validated_env('TEST_ENV', check_symlink=True), '/path/to/file') + +if __name__ == '__main__': + unittest.main() \ No newline at end of file -- Gitee From 73fa9cef09be55a64cf91e683ef8df088a34f7da Mon Sep 17 00:00:00 2001 From: shenlian Date: Tue, 1 Jul 2025 11:44:30 +0800 Subject: [PATCH 04/10] add env check --- ascend_deployer/downloader/download_util.py | 8 +- ascend_deployer/downloader/logger_config.py | 19 ++- .../large_scale_deploy/tools/log_tool.py | 6 +- .../check_library_utils/npu_checks.py | 15 ++ ascend_deployer/utils.py | 149 +++++++++--------- 5 files changed, 107 insertions(+), 90 deletions(-) diff --git a/ascend_deployer/downloader/download_util.py b/ascend_deployer/downloader/download_util.py index 561fdbc5..43d1d525 100644 --- a/ascend_deployer/downloader/download_util.py +++ b/ascend_deployer/downloader/download_util.py @@ -30,7 +30,7 @@ from pathlib import PurePath from urllib import request from urllib.error import ContentTooShortError, URLError from typing import Optional - +from ascend_deployer.utils import get_validated_env from . import logger_config REFERER = "https://www.hiascend.com/" @@ -89,9 +89,9 @@ def get_download_path(): return cur if platform.system() == 'Linux': - deployer_home = os.getenv('HOME') - if os.getenv('ASCEND_DEPLOYER_HOME') is not None: - deployer_home = os.getenv('ASCEND_DEPLOYER_HOME') + deployer_home = get_validated_env('HOME') + if get_validated_env('ASCEND_DEPLOYER_HOME') is not None: + deployer_home = get_validated_env('ASCEND_DEPLOYER_HOME') else: deployer_home = os.getcwd() diff --git a/ascend_deployer/downloader/logger_config.py b/ascend_deployer/downloader/logger_config.py index 7544e387..915c15f5 100644 --- a/ascend_deployer/downloader/logger_config.py +++ b/ascend_deployer/downloader/logger_config.py @@ -20,12 +20,14 @@ import logging.handlers import os import platform import stat +from ascend_deployer.utils import get_validated_env class RotatingFileHandler(logging.handlers.RotatingFileHandler): """ rewrite RotatingFileHandler, assign permissions to downloader.log and downloader.log.* """ + def doRollover(self): largest_backfile = "{}.{}".format(self.baseFilename, self.backupCount) if os.path.exists(largest_backfile): @@ -48,9 +50,9 @@ class BasicLogConfig(object): else: deployer_home = '' if platform.system() == 'Linux': - deployer_home = os.getenv('HOME') - if os.getenv('ASCEND_DEPLOYER_HOME') is not None: - deployer_home = os.getenv('ASCEND_DEPLOYER_HOME') + deployer_home = get_validated_env('HOME') + if get_validated_env('ASCEND_DEPLOYER_HOME') is not None: + deployer_home = get_validated_env('ASCEND_DEPLOYER_HOME') else: deployer_home = os.getcwd() parent_dir = os.path.join(deployer_home, 'ascend-deployer') @@ -71,15 +73,15 @@ class BasicLogConfig(object): os.chmod(LOG_FILE_OPERATION, stat.S_IRUSR | stat.S_IWUSR) USER_NAME = getpass.getuser() - CLIENT_IP = os.getenv('SSH_CLIENT', 'localhost').split()[0] + CLIENT_IP = (get_validated_env('SSH_CLIENT') or 'localhost').split()[0] EXTRA = {'user_name': USER_NAME, 'client_ip': CLIENT_IP} LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' LOG_FORMAT_STRING = \ - "%(asctime)s downloader [%(levelname)s] " \ - "[%(filename)s:%(lineno)d %(funcName)s] %(message)s" + "%(asctime)s downloader [%(levelname)s] " \ + "[%(filename)s:%(lineno)d %(funcName)s] %(message)s" LOG_FORMAT_STRING_OPERATION = \ - "%(asctime)s localhost [%(levelname)s] " \ - "[%(filename)s:%(lineno)d %(funcName)s] %(message)s" + "%(asctime)s localhost [%(levelname)s] " \ + "[%(filename)s:%(lineno)d %(funcName)s] %(message)s" LOG_LEVEL = logging.INFO ROTATING_CONF = dict( @@ -88,6 +90,7 @@ class BasicLogConfig(object): backupCount=5, encoding="UTF-8") + LOG_CONF = BasicLogConfig() diff --git a/ascend_deployer/large_scale_deploy/tools/log_tool.py b/ascend_deployer/large_scale_deploy/tools/log_tool.py index 2fe6e8b0..999e8a2d 100644 --- a/ascend_deployer/large_scale_deploy/tools/log_tool.py +++ b/ascend_deployer/large_scale_deploy/tools/log_tool.py @@ -3,7 +3,7 @@ import logging.handlers import os import stat import sys - +from ascend_deployer.utils import get_validated_env CUR_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -14,8 +14,8 @@ class LogTool: if 'site-packages' not in CUR_DIR and 'dist-packages' not in CUR_DIR: log_dir = os.path.dirname(CUR_DIR) else: - if os.getenv('ASCEND_DEPLOYER_HOME'): - deployer_home = os.getenv('ASCEND_DEPLOYER_HOME') + if get_validated_env('ASCEND_DEPLOYER_HOME'): + deployer_home = get_validated_env('ASCEND_DEPLOYER_HOME') else: deployer_home = os.getcwd() log_dir = os.path.join(deployer_home, "ascend-deployer") diff --git a/ascend_deployer/module_utils/check_library_utils/npu_checks.py b/ascend_deployer/module_utils/check_library_utils/npu_checks.py index b931959b..defafc45 100644 --- a/ascend_deployer/module_utils/check_library_utils/npu_checks.py +++ b/ascend_deployer/module_utils/check_library_utils/npu_checks.py @@ -199,3 +199,18 @@ class NPUCheck: util.record_error("[ASCEND][[ERROR]] Critical issue with NPU, please check the health of card.", self.error_messages) return + +金雪 +王虎 +助理 +田如石 +茶艳 +小熊 +阳姐 +天奇 +二红 +梦婷 +龙 +张阳老婆 +陈通 +龙中浪 \ No newline at end of file diff --git a/ascend_deployer/utils.py b/ascend_deployer/utils.py index 89cb2088..70e7d41c 100644 --- a/ascend_deployer/utils.py +++ b/ascend_deployer/utils.py @@ -46,6 +46,78 @@ file_list = ['install.sh', 'inventory_file', 'ansible.cfg', 'version.json'] VERSION_PATTERN = re.compile(r"(\d+)") +PATH_WHITE_LIST_LIN = string.digits + string.ascii_letters + '~-+_./ ' +MAX_PATH_LEN = 4096 + + +def get_validated_env( + env_name, + whitelist=PATH_WHITE_LIST_LIN, + min_length=1, + max_length=MAX_PATH_LEN, + check_symlink=True +): + """ + 获取并验证环境变量 (兼容 Python 2/3) + :param env_name: 环境变量名称 + :param whitelist: 允许的值列表 + :param min_length: 最小长度限制 + :param max_length: 最大长度限制 + :param check_symlink: 是否检查软链接 + :return: 验证通过的环境变量值 + :raises ValueError: 验证失败时抛出 + """ + value = os.getenv(env_name) + + if value is None: + return None + + # 白名单校验 + for char in value: + if char not in whitelist: + raise ValueError( + "The path is invalid. The path can contain only char in '{}'".format(whitelist)) + + # 长度校验 + str_len = len(value) + if min_length is not None and str_len < min_length: + raise ValueError( + "Value for {} is too short. Minimum length: {}, actual: {}".format( + env_name, min_length, str_len + ) + ) + + if max_length is not None and str_len > max_length: + raise ValueError( + "Value for {} is too long. Maximum length: {}, actual: {}".format( + env_name, max_length, str_len + ) + ) + + # 路径安全校验 + if check_symlink: + # 在 Python 2/3 中正确处理 unicode 路径 + if isinstance(value, bytes): + path_value = value.decode('utf-8', 'replace') + else: + path_value = value + # 软链接检查 + if check_symlink: + try: + # 检查路径是否存在且是符号链接 + if os.path.lexists(path_value) and os.path.islink(path_value): + raise ValueError( + "Path for {} is a symlink: {}. Symlinks are not allowed for security reasons.".format( + env_name, path_value + ) + ) + except (OSError, IOError) as e: + # 处理文件系统访问错误 + if e.errno != errno.ENOENT: # 忽略文件不存在的错误 + raise ValueError( + "Error checking symlink for {}: {} - {}".format(env_name, path_value, str(e)) + ) + return value def compare_version(src_version, target_version): @@ -87,7 +159,7 @@ def copy_scripts(): if 'site-packages' in ROOT_PATH or 'dist-packages' in ROOT_PATH: deployer_home = os.getcwd() if platform.system() == 'Linux': - deployer_home = os.getenv('ASCEND_DEPLOYER_HOME', os.getenv('HOME')) + deployer_home = get_validated_env('ASCEND_DEPLOYER_HOME') or get_validated_env('HOME') ROOT_PATH = os.path.join(deployer_home, 'ascend-deployer') copy_scripts() @@ -174,7 +246,7 @@ LOG_OPERATION_FILE = os.path.join(ROOT_PATH, 'install_operation.log') class UserHostFilter(logging.Filter): user = getpass.getuser() - host = os.getenv('SSH_CLIENT', 'localhost').split()[0] + host = (get_validated_env('SSH_CLIENT') or 'localhost').split()[0] def filter(self, record): record.user = self.user @@ -306,76 +378,3 @@ def get_hosts_name(tags): if (isinstance(tags, str) and tags in dl_items) or (isinstance(tags, list) and set(tags) & set(dl_items)): return 'master,worker' return 'worker' - - - -PATH_WHITE_LIST_LIN = string.digits + string.ascii_letters + '~-+_./ ' -MAX_PATH_LEN = 4096 -def get_validated_env( - env_name, - whitelist=PATH_WHITE_LIST_LIN, - min_length=1, - max_length=MAX_PATH_LEN, - check_symlink=True -): - """ - 获取并验证环境变量 (兼容 Python 2/3) - :param env_name: 环境变量名称 - :param whitelist: 允许的值列表 - :param min_length: 最小长度限制 - :param max_length: 最大长度限制 - :param check_symlink: 是否检查软链接 - :return: 验证通过的环境变量值 - :raises ValueError: 验证失败时抛出 - """ - value = os.getenv(env_name) - - if value is None: - return None - - # 白名单校验 - for char in value: - if char not in whitelist: - raise ValueError( - "The path is invalid. The path can contain only char in '{}'".format(whitelist)) - - # 长度校验 - str_len = len(value) - if min_length is not None and str_len < min_length: - raise ValueError( - "Value for {} is too short. Minimum length: {}, actual: {}".format( - env_name, min_length, str_len - ) - ) - - if max_length is not None and str_len > max_length: - raise ValueError( - "Value for {} is too long. Maximum length: {}, actual: {}".format( - env_name, max_length, str_len - ) - ) - - # 路径安全校验 - if check_symlink: - # 在 Python 2/3 中正确处理 unicode 路径 - if isinstance(value, bytes): - path_value = value.decode('utf-8', 'replace') - else: - path_value = value - # 软链接检查 - if check_symlink: - try: - # 检查路径是否存在且是符号链接 - if os.path.lexists(path_value) and os.path.islink(path_value): - raise ValueError( - "Path for {} is a symlink: {}. Symlinks are not allowed for security reasons.".format( - env_name, path_value - ) - ) - except (OSError, IOError) as e: - # 处理文件系统访问错误 - if e.errno != errno.ENOENT: # 忽略文件不存在的错误 - raise ValueError( - "Error checking symlink for {}: {} - {}".format(env_name, path_value, str(e)) - ) - return value \ No newline at end of file -- Gitee From de47c261d8c648dbf679e4898f941404331f5063 Mon Sep 17 00:00:00 2001 From: shenlian Date: Tue, 1 Jul 2025 14:36:25 +0800 Subject: [PATCH 05/10] add env check --- ascend_deployer/downloader/download_util.py | 2 +- ascend_deployer/downloader/logger_config.py | 2 +- ascend_deployer/large_scale_deploy/tools/log_tool.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ascend_deployer/downloader/download_util.py b/ascend_deployer/downloader/download_util.py index 43d1d525..6190cd67 100644 --- a/ascend_deployer/downloader/download_util.py +++ b/ascend_deployer/downloader/download_util.py @@ -30,7 +30,7 @@ from pathlib import PurePath from urllib import request from urllib.error import ContentTooShortError, URLError from typing import Optional -from ascend_deployer.utils import get_validated_env +from ..utils import get_validated_env from . import logger_config REFERER = "https://www.hiascend.com/" diff --git a/ascend_deployer/downloader/logger_config.py b/ascend_deployer/downloader/logger_config.py index 915c15f5..721b27b9 100644 --- a/ascend_deployer/downloader/logger_config.py +++ b/ascend_deployer/downloader/logger_config.py @@ -20,7 +20,7 @@ import logging.handlers import os import platform import stat -from ascend_deployer.utils import get_validated_env +from ..utils import get_validated_env class RotatingFileHandler(logging.handlers.RotatingFileHandler): diff --git a/ascend_deployer/large_scale_deploy/tools/log_tool.py b/ascend_deployer/large_scale_deploy/tools/log_tool.py index 999e8a2d..6ae72fec 100644 --- a/ascend_deployer/large_scale_deploy/tools/log_tool.py +++ b/ascend_deployer/large_scale_deploy/tools/log_tool.py @@ -3,7 +3,7 @@ import logging.handlers import os import stat import sys -from ascend_deployer.utils import get_validated_env +from ...utils import get_validated_env CUR_DIR = os.path.dirname(os.path.realpath(__file__)) -- Gitee From 1923bedbd6785a437b87efef17089222a40dbcad Mon Sep 17 00:00:00 2001 From: shenlian Date: Tue, 1 Jul 2025 15:03:39 +0800 Subject: [PATCH 06/10] add env check --- ascend_deployer/downloader/download_util.py | 2 +- ascend_deployer/downloader/logger_config.py | 2 +- .../large_scale_deploy/tools/log_tool.py | 2 +- ascend_deployer/module_utils/path_manager.py | 74 ++++++++++++++++++ ascend_deployer/utils.py | 76 +------------------ 5 files changed, 78 insertions(+), 78 deletions(-) diff --git a/ascend_deployer/downloader/download_util.py b/ascend_deployer/downloader/download_util.py index 6190cd67..f5009250 100644 --- a/ascend_deployer/downloader/download_util.py +++ b/ascend_deployer/downloader/download_util.py @@ -30,7 +30,7 @@ from pathlib import PurePath from urllib import request from urllib.error import ContentTooShortError, URLError from typing import Optional -from ..utils import get_validated_env +from ..module_utils.path_manager import get_validated_env from . import logger_config REFERER = "https://www.hiascend.com/" diff --git a/ascend_deployer/downloader/logger_config.py b/ascend_deployer/downloader/logger_config.py index 721b27b9..73872c57 100644 --- a/ascend_deployer/downloader/logger_config.py +++ b/ascend_deployer/downloader/logger_config.py @@ -20,7 +20,7 @@ import logging.handlers import os import platform import stat -from ..utils import get_validated_env +from ..module_utils.path_manager import get_validated_env class RotatingFileHandler(logging.handlers.RotatingFileHandler): diff --git a/ascend_deployer/large_scale_deploy/tools/log_tool.py b/ascend_deployer/large_scale_deploy/tools/log_tool.py index 6ae72fec..539bb51a 100644 --- a/ascend_deployer/large_scale_deploy/tools/log_tool.py +++ b/ascend_deployer/large_scale_deploy/tools/log_tool.py @@ -3,7 +3,7 @@ import logging.handlers import os import stat import sys -from ...utils import get_validated_env +from ...module_utils.path_manager import get_validated_env CUR_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/ascend_deployer/module_utils/path_manager.py b/ascend_deployer/module_utils/path_manager.py index c16b3a80..32b7ce6f 100644 --- a/ascend_deployer/module_utils/path_manager.py +++ b/ascend_deployer/module_utils/path_manager.py @@ -1,7 +1,81 @@ +import errno import os.path import shutil +import string _CUR_DIR = os.path.dirname(__file__) +PATH_WHITE_LIST_LIN = string.digits + string.ascii_letters + '~-+_./ ' +MAX_PATH_LEN = 4096 + + +def get_validated_env( + env_name, + whitelist=PATH_WHITE_LIST_LIN, + min_length=1, + max_length=MAX_PATH_LEN, + check_symlink=True +): + """ + 获取并验证环境变量 (兼容 Python 2/3) + :param env_name: 环境变量名称 + :param whitelist: 允许的值列表 + :param min_length: 最小长度限制 + :param max_length: 最大长度限制 + :param check_symlink: 是否检查软链接 + :return: 验证通过的环境变量值 + :raises ValueError: 验证失败时抛出 + """ + value = os.getenv(env_name) + + if value is None: + return None + + # 白名单校验 + for char in value: + if char not in whitelist: + raise ValueError( + "The path is invalid. The path can contain only char in '{}'".format(whitelist)) + + # 长度校验 + str_len = len(value) + if min_length is not None and str_len < min_length: + raise ValueError( + "Value for {} is too short. Minimum length: {}, actual: {}".format( + env_name, min_length, str_len + ) + ) + + if max_length is not None and str_len > max_length: + raise ValueError( + "Value for {} is too long. Maximum length: {}, actual: {}".format( + env_name, max_length, str_len + ) + ) + + # 路径安全校验 + if check_symlink: + # 在 Python 2/3 中正确处理 unicode 路径 + if isinstance(value, bytes): + path_value = value.decode('utf-8', 'replace') + else: + path_value = value + # 软链接检查 + if check_symlink: + try: + # 检查路径是否存在且是符号链接 + if os.path.lexists(path_value) and os.path.islink(path_value): + raise ValueError( + "Path for {} is a symlink: {}. Symlinks are not allowed for security reasons.".format( + env_name, path_value + ) + ) + except (OSError, IOError) as e: + # 处理文件系统访问错误 + if e.errno != errno.ENOENT: # 忽略文件不存在的错误 + raise ValueError( + "Error checking symlink for {}: {} - {}".format(env_name, path_value, str(e)) + ) + return value class ProjectPath: diff --git a/ascend_deployer/utils.py b/ascend_deployer/utils.py index 70e7d41c..9955f168 100644 --- a/ascend_deployer/utils.py +++ b/ascend_deployer/utils.py @@ -17,7 +17,6 @@ import json import shlex import stat -import string import argparse import getpass import logging @@ -27,9 +26,8 @@ import shutil import re import os import sys -import errno from subprocess import PIPE, Popen - +from module_utils.path_manager import get_validated_env ROOT_PATH = SRC_PATH = os.path.dirname(__file__) NEXUS_SENTINEL_FILE = os.path.expanduser('~/.local/nexus.sentinel') MODE_700 = stat.S_IRWXU @@ -46,78 +44,6 @@ file_list = ['install.sh', 'inventory_file', 'ansible.cfg', 'version.json'] VERSION_PATTERN = re.compile(r"(\d+)") -PATH_WHITE_LIST_LIN = string.digits + string.ascii_letters + '~-+_./ ' -MAX_PATH_LEN = 4096 - - -def get_validated_env( - env_name, - whitelist=PATH_WHITE_LIST_LIN, - min_length=1, - max_length=MAX_PATH_LEN, - check_symlink=True -): - """ - 获取并验证环境变量 (兼容 Python 2/3) - :param env_name: 环境变量名称 - :param whitelist: 允许的值列表 - :param min_length: 最小长度限制 - :param max_length: 最大长度限制 - :param check_symlink: 是否检查软链接 - :return: 验证通过的环境变量值 - :raises ValueError: 验证失败时抛出 - """ - value = os.getenv(env_name) - - if value is None: - return None - - # 白名单校验 - for char in value: - if char not in whitelist: - raise ValueError( - "The path is invalid. The path can contain only char in '{}'".format(whitelist)) - - # 长度校验 - str_len = len(value) - if min_length is not None and str_len < min_length: - raise ValueError( - "Value for {} is too short. Minimum length: {}, actual: {}".format( - env_name, min_length, str_len - ) - ) - - if max_length is not None and str_len > max_length: - raise ValueError( - "Value for {} is too long. Maximum length: {}, actual: {}".format( - env_name, max_length, str_len - ) - ) - - # 路径安全校验 - if check_symlink: - # 在 Python 2/3 中正确处理 unicode 路径 - if isinstance(value, bytes): - path_value = value.decode('utf-8', 'replace') - else: - path_value = value - # 软链接检查 - if check_symlink: - try: - # 检查路径是否存在且是符号链接 - if os.path.lexists(path_value) and os.path.islink(path_value): - raise ValueError( - "Path for {} is a symlink: {}. Symlinks are not allowed for security reasons.".format( - env_name, path_value - ) - ) - except (OSError, IOError) as e: - # 处理文件系统访问错误 - if e.errno != errno.ENOENT: # 忽略文件不存在的错误 - raise ValueError( - "Error checking symlink for {}: {} - {}".format(env_name, path_value, str(e)) - ) - return value def compare_version(src_version, target_version): -- Gitee From 105d2846b403a30be0af9570d23afac2e3dad1fd Mon Sep 17 00:00:00 2001 From: shenlian Date: Tue, 1 Jul 2025 15:04:55 +0800 Subject: [PATCH 07/10] add env check --- .../check_library_utils/npu_checks.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/ascend_deployer/module_utils/check_library_utils/npu_checks.py b/ascend_deployer/module_utils/check_library_utils/npu_checks.py index defafc45..d0f83876 100644 --- a/ascend_deployer/module_utils/check_library_utils/npu_checks.py +++ b/ascend_deployer/module_utils/check_library_utils/npu_checks.py @@ -198,19 +198,4 @@ class NPUCheck: if status != "|" and status != "OK": util.record_error("[ASCEND][[ERROR]] Critical issue with NPU, please check the health of card.", self.error_messages) - return - -金雪 -王虎 -助理 -田如石 -茶艳 -小熊 -阳姐 -天奇 -二红 -梦婷 -龙 -张阳老婆 -陈通 -龙中浪 \ No newline at end of file + return \ No newline at end of file -- Gitee From c06d091ad0708a89921963c74ea4270f3ea43f11 Mon Sep 17 00:00:00 2001 From: shenlian Date: Wed, 2 Jul 2025 11:10:47 +0800 Subject: [PATCH 08/10] add env check --- ascend_deployer/{ascend_deployer.py => ascend.py} | 0 ascend_deployer/downloader/download_util.py | 2 +- ascend_deployer/downloader/logger_config.py | 2 +- ascend_deployer/install.sh | 4 ++-- ascend_deployer/large_scale_deploy/tools/log_tool.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename ascend_deployer/{ascend_deployer.py => ascend.py} (100%) diff --git a/ascend_deployer/ascend_deployer.py b/ascend_deployer/ascend.py similarity index 100% rename from ascend_deployer/ascend_deployer.py rename to ascend_deployer/ascend.py diff --git a/ascend_deployer/downloader/download_util.py b/ascend_deployer/downloader/download_util.py index f5009250..e306252f 100644 --- a/ascend_deployer/downloader/download_util.py +++ b/ascend_deployer/downloader/download_util.py @@ -30,7 +30,7 @@ from pathlib import PurePath from urllib import request from urllib.error import ContentTooShortError, URLError from typing import Optional -from ..module_utils.path_manager import get_validated_env +from ascend_deployer.module_utils.path_manager import get_validated_env from . import logger_config REFERER = "https://www.hiascend.com/" diff --git a/ascend_deployer/downloader/logger_config.py b/ascend_deployer/downloader/logger_config.py index 73872c57..4244df79 100644 --- a/ascend_deployer/downloader/logger_config.py +++ b/ascend_deployer/downloader/logger_config.py @@ -20,7 +20,7 @@ import logging.handlers import os import platform import stat -from ..module_utils.path_manager import get_validated_env +from ascend_deployer.module_utils.path_manager import get_validated_env class RotatingFileHandler(logging.handlers.RotatingFileHandler): diff --git a/ascend_deployer/install.sh b/ascend_deployer/install.sh index db15292f..49786cf1 100644 --- a/ascend_deployer/install.sh +++ b/ascend_deployer/install.sh @@ -10,9 +10,9 @@ main() { fi python3 -V > /dev/null 2>&1 if [[ $? != 0 ]]; then - python ${BASE_DIR}/ascend_deployer.py $* + python ${BASE_DIR}/ascend.py $* else - python3 ${BASE_DIR}/ascend_deployer.py $* + python3 ${BASE_DIR}/ascend.py $* fi } diff --git a/ascend_deployer/large_scale_deploy/tools/log_tool.py b/ascend_deployer/large_scale_deploy/tools/log_tool.py index 539bb51a..48c48b38 100644 --- a/ascend_deployer/large_scale_deploy/tools/log_tool.py +++ b/ascend_deployer/large_scale_deploy/tools/log_tool.py @@ -3,7 +3,7 @@ import logging.handlers import os import stat import sys -from ...module_utils.path_manager import get_validated_env +from ascend_deployer.module_utils.path_manager import get_validated_env CUR_DIR = os.path.dirname(os.path.realpath(__file__)) -- Gitee From 662a0e3ce4e79ccd385c62c4431483a415e14dfa Mon Sep 17 00:00:00 2001 From: shenlian Date: Wed, 2 Jul 2025 12:09:12 +0800 Subject: [PATCH 09/10] add env check --- ascend_deployer/install.sh | 4 ++-- ascend_deployer/{ascend.py => start_deploy.py} | 0 setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename ascend_deployer/{ascend.py => start_deploy.py} (100%) diff --git a/ascend_deployer/install.sh b/ascend_deployer/install.sh index 49786cf1..0c3b8e71 100644 --- a/ascend_deployer/install.sh +++ b/ascend_deployer/install.sh @@ -10,9 +10,9 @@ main() { fi python3 -V > /dev/null 2>&1 if [[ $? != 0 ]]; then - python ${BASE_DIR}/ascend.py $* + python ${BASE_DIR}/start_deploy.py $* else - python3 ${BASE_DIR}/ascend.py $* + python3 ${BASE_DIR}/start_deploy.py $* fi } diff --git a/ascend_deployer/ascend.py b/ascend_deployer/start_deploy.py similarity index 100% rename from ascend_deployer/ascend.py rename to ascend_deployer/start_deploy.py diff --git a/setup.py b/setup.py index 001eb2ad..6e540c5a 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ setuptools.setup( entry_points={ # Optional 'console_scripts': [ 'ascend-download=ascend_deployer.ascend_download:main', - 'ascend-deployer=ascend_deployer.ascend_deployer:main', + 'ascend-deployer=ascend_deployer.start_deploy:main', 'large-scale-deployer=ascend_deployer.large_scale_deployer:main' ] }, -- Gitee From 053a7f996944ffc2acd20fbee56878c2f1fef667 Mon Sep 17 00:00:00 2001 From: shenlian Date: Wed, 2 Jul 2025 15:06:51 +0800 Subject: [PATCH 10/10] add env check --- ascend_deployer/downloader/download_util.py | 2 ++ ascend_deployer/downloader/logger_config.py | 4 ++++ ascend_deployer/large_scale_deploy/tools/log_tool.py | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/ascend_deployer/downloader/download_util.py b/ascend_deployer/downloader/download_util.py index e306252f..cb7a2b5a 100644 --- a/ascend_deployer/downloader/download_util.py +++ b/ascend_deployer/downloader/download_util.py @@ -30,6 +30,8 @@ from pathlib import PurePath from urllib import request from urllib.error import ContentTooShortError, URLError from typing import Optional +base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, base_dir) from ascend_deployer.module_utils.path_manager import get_validated_env from . import logger_config diff --git a/ascend_deployer/downloader/logger_config.py b/ascend_deployer/downloader/logger_config.py index 4244df79..641c052d 100644 --- a/ascend_deployer/downloader/logger_config.py +++ b/ascend_deployer/downloader/logger_config.py @@ -20,6 +20,10 @@ import logging.handlers import os import platform import stat +import sys + +base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, base_dir) from ascend_deployer.module_utils.path_manager import get_validated_env diff --git a/ascend_deployer/large_scale_deploy/tools/log_tool.py b/ascend_deployer/large_scale_deploy/tools/log_tool.py index 48c48b38..0e2918dc 100644 --- a/ascend_deployer/large_scale_deploy/tools/log_tool.py +++ b/ascend_deployer/large_scale_deploy/tools/log_tool.py @@ -3,7 +3,11 @@ import logging.handlers import os import stat import sys + +base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +sys.path.insert(0, base_dir) from ascend_deployer.module_utils.path_manager import get_validated_env + CUR_DIR = os.path.dirname(os.path.realpath(__file__)) -- Gitee