diff --git a/omniadvisor/compile.py b/omniadvisor/compile.py index 11b4fb18b2a4becd12f1afa39cc90944dcddbf61..8cf90de6d4dc513a38b53f3e2c0d598a43474939 100644 --- a/omniadvisor/compile.py +++ b/omniadvisor/compile.py @@ -2,89 +2,99 @@ # Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. """ """ +import fnmatch import os -import compileall import shutil +import subprocess -# 要编译的文件夹的名称 -TO_COMPLIES = ['src'] -# 要保留的文件夹的名称 -TO_RETAINS = ['config'] -# 编译文件的输出路径 -OUTPUT_DIR_NAME = 'BoostKit-omniadvisor_2.0' -# 本文件的路径 -BASE_PATH = os.path.dirname(os.path.abspath(__file__)) +PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) +# 与src同级的其他资源文件夹 +RESOURCES_LIST = ['config'] -def clean_pyc_files(project_dir): - """删除旧的 .pyc 文件""" - for root, _, files in os.walk(project_dir): - for file in files: - if file.endswith(".pyc"): - pyc_path = os.path.join(root, file) - os.remove(pyc_path) +# 要编译的源文件夹的名称 +COMPILE_FROM_NAME = 'src' +# 要编译的源码目录，一般就是 src +SOURCE_DIR = os.path.join(PROJECT_DIR, COMPILE_FROM_NAME) -def compile_python_files(project_dir): - """编译所有 .py 文件为 .pyc""" - compileall.compile_dir(project_dir, force=True, quiet=1) +# 编译后输出的根文件夹名称 +COMPILE_DEST_ROOT_NAME = 'BoostKit-omniadvisor_2.0' +# 编译后输出的根目录 +OUTPUT_ROOT_DIR = os.path.join(PROJECT_DIR, COMPILE_DEST_ROOT_NAME) -def replace_pyc_files(project_dir): - """将 __pycache__ 中的新 .pyc 移动到对应位置，替换旧的 .pyc""" - for root, dirs, _ in os.walk(project_dir): # 深度 1 - if "__pycache__" not in dirs: # 提前跳过不相关的目录 - continue # 减少一层缩进 +# 编译后输出的目录 +OUTPUT_SRC_DIR = os.path.join(OUTPUT_ROOT_DIR, COMPILE_FROM_NAME) - pycache_path = os.path.join(root, "__pycache__") - for file in os.listdir(pycache_path): # 深度 2 - if not file.endswith(".pyc"): # 跳过非 .pyc 文件 - continue # 减少一层缩进 +# 原样保留的文件 +RETAIN_FILES = ['*.so', '*.json', '*.cfg', '*.pyc'] - # 目标 .py 文件名 - module_name = file.partition(".")[0] # 更安全的方式提取模块名 - if not module_name: # 避免空模块名 - continue # 减少一层缩进 +# 这个是产生的临时的编译目录 +TEMP_BUILD_DIR = os.path.join(OUTPUT_SRC_DIR, 'build') - target_file = os.path.join(root, f"{module_name}.pyc") - source_pyc = os.path.join(pycache_path, file) - shutil.move(source_pyc, target_file) # 深度 2 - # 可选：删除空 __pycache__ 目录 - if not os.listdir(pycache_path): # 深度 2 - os.rmdir(pycache_path) # 深度 3 +def delete_files_recursively(root_dir, patterns): + """ + 递归删除指定目录下符合特定模式的文件 + :param root_dir: 要搜索的根目录 + :param patterns: 要删除的文件模式列表 + """ + # 保留必要文件，其他删除 + for dirpath, _, filenames in os.walk(root_dir): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + # 检查文件名是否匹配任一模式 + if (filename in patterns or + any(fnmatch.fnmatch(filename, pattern) for pattern in patterns)): + continue + try: + os.remove(filepath) + except OSError as e: + raise RuntimeError(f"删除 {filepath} 失败 ") from e -def remove_source_files(project_dir): - # 遍历文件夹 - for root, _, files in os.walk(project_dir): - for file in files: - # 检查文件扩展名是否为 .py - if file.endswith(".py"): - file_path = os.path.join(root, file) - os.remove(file_path) + # 重命名，默认是 utils.cpython-38-aarch64-linux-gnu.so，即 模块名.xxx.so + for root, _, files in os.walk(root_dir): + for filename in files: + if not filename.endswith('.so'): + continue + parts = filename.split('.') + if len(parts) >= 3: # 确保文件名符合 xxx.yyy.so 格式 + new_name = f"{parts[0]}.so" # 取第一个和最后一个部分 + old_path = os.path.join(root, filename) + new_path = os.path.join(root, new_name) -def rebuild_pyc_files(project_dir): - clean_pyc_files(project_dir) - compile_python_files(project_dir) - replace_pyc_files(project_dir) - remove_source_files(project_dir) + os.rename(old_path, new_path) + # 清理build目录 + if os.path.exists(TEMP_BUILD_DIR): + shutil.rmtree(TEMP_BUILD_DIR) -if __name__ == "__main__": - # 将此处替换为你的项目路径 - output_path = os.path.join(BASE_PATH, OUTPUT_DIR_NAME) - # 删除整个文件夹及其内容 - if os.path.exists(output_path): - shutil.rmtree(output_path) +if __name__ == '__main__': + if os.path.exists(OUTPUT_ROOT_DIR): + shutil.rmtree(OUTPUT_ROOT_DIR) - # 创建文件夹 - os.makedirs(output_path) + shutil.copytree(SOURCE_DIR, OUTPUT_SRC_DIR) + for resource in RESOURCES_LIST: + if os.path.exists(os.path.join(PROJECT_DIR, resource)): + shutil.copytree(os.path.join(PROJECT_DIR, resource), os.path.join(OUTPUT_ROOT_DIR, resource)) - # copy 需要编译的目录 - for name in TO_RETAINS + TO_COMPLIES: - shutil.copytree(os.path.join(BASE_PATH, name), os.path.join(output_path, name)) + # 进入 build_output 目录 + original_dir = os.getcwd() + os.chdir(OUTPUT_SRC_DIR) - rebuild_pyc_files(os.path.join(output_path, TO_COMPLIES[0])) + arg_list = ["python3", "setup.py", "build_ext", "--inplace"] + + try: + # 这里可以根据你的实际打包命令进行修改 + subprocess.run(arg_list, check=True) + + finally: + # 返回原始目录 + os.chdir(original_dir) + + # 删除build_output 目录下所有无用文件 + delete_files_recursively(OUTPUT_SRC_DIR, RETAIN_FILES) diff --git a/omniadvisor/pyproject.toml b/omniadvisor/pyproject.toml index 37dde83d99e035ec5763e80d252601dcf4c7037f..94b80b53c14b35c5fdb3a416c398ae8eeb419893 100755 --- a/omniadvisor/pyproject.toml +++ b/omniadvisor/pyproject.toml @@ -24,6 +24,7 @@ requests = "^2.32.3" [tool.poetry.group.test.dependencies] pytest = "^7.4.4" pytest-xdist = "^3.6.1" +Cython = "^3.0.10" [tool.pytest.ini_options] testpaths = ["tests/"] # 指定 pytest 搜索测试的路径 diff --git a/omniadvisor/src/setup.py b/omniadvisor/src/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..ecdb8ddfd7eedc3bb627ee6282945ee0f7573719 --- /dev/null +++ b/omniadvisor/src/setup.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Huawei Technologies Co., Ltd. 2025-2025. All rights reserved. +""" +""" +import os +import py_compile + +from Cython.Build import cythonize +from setuptools import setup, Extension, find_packages + +# 要编译成 .pyc的文件（几个入口文件） +pyc_files = ['__init__.py', 'init.py', 'hijack.py', 'tuning.py'] + + +# 1 需要编译的文件 +def find_py_files(root): + py_files = [] + for dirpath, _, filenames in os.walk(root): + for f in filenames: + if not f.endswith(".py") or f == 'setup.py': + continue + if f in pyc_files: + # 编译成 .pyc，并删除原 .py文件 + py_path = os.path.join(dirpath, f) + # 编译生成.pyc文件 + py_compile.compile(py_path, cfile=py_path + 'c') + # 删除原.py文件 + os.remove(py_path) + continue + py_files.append(os.path.join(dirpath, f)) + return py_files + + +# 配置扩展模块 +extensions = [] +for py_file in find_py_files("."): + # 将路径转换为模块名，如 iterative.config_space.module1 + module_name = py_file[:-3].replace('./', '').replace(os.path.sep, ".") + + extensions.append( + Extension( + module_name, + [py_file], + extra_compile_args=["-O3"], # 优化选项 + ) + ) + +# 编译配置 +setup( + name="show demo", + # 自动发现所有包（包含 __init__.py 的目录） + packages=find_packages(include=['.*']), + # 包含所有JSON文件 + ext_modules=cythonize( + extensions, + # 使用 Python 3；并禁用类型检查，以避免不必要的报错 重要！勿删！ + compiler_directives={"language_level": "3", "annotation_typing": False}, + build_dir="build", + ), + script_args=["build_ext", "--inplace"], # 编译到源码目录 +) \ No newline at end of file