diff --git a/.cid/__init__.py b/.cid/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.cid/build.py b/.cid/build.py index 5f05fbc2e7dd14247436b0b38f245f85f6cc238a..c8c8652319ccc4b30f4602e672d2c6d20e0767ed 100644 --- a/.cid/build.py +++ b/.cid/build.py @@ -19,7 +19,10 @@ import logging import shutil from pathlib import Path -from python_builder import MinGWPythonBuilder, BuildConfig +from python_builder import BuildConfig +from darwin_python_builder import DarwinPythonBuilder +from linux_python_builder import LinuxPythonBuilder +from mingw_python_builder import MinGWPythonBuilder def main(): @@ -27,8 +30,8 @@ def main(): parser.add_argument('--repo-root', default='.', help='Repository root directory') parser.add_argument('--out-path', default='./out', help='Output directory') parser.add_argument('--lldb-py-version', default='3.11.4', help='LLDB Python version') - parser.add_argument('--lldb-py-detailed-version', default='3.11.4_20250509', help='LLDB Python detailed version') - parser.add_argument('--mingw-triple', default='x86_64-w64-mingw32', help='MinGW triple') + parser.add_argument('--target-os', default='linux', help='Target OS') + parser.add_argument('--target-arch', default='x86', help='Target architecture') args = parser.parse_args() build_config = BuildConfig(args) @@ -41,15 +44,20 @@ def main(): logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', filename=build_config.OUT_PATH + '/build.log') - + try: - mingw_builder = MinGWPythonBuilder(build_config) - mingw_builder.build() - mingw_builder.prepare_for_package() - mingw_builder.package() - logging.info("MinGW Python 构建、准备和打包完成。") + if args.target_os == 'linux': + python_builder = LinuxPythonBuilder(build_config) + elif args.target_os == 'mingw': + python_builder = MinGWPythonBuilder(build_config) + elif args.target_os == 'darwin': + python_builder = DarwinPythonBuilder(build_config) + else: + raise ValueError(f"Unsupported target OS: {args.target_os}") + python_builder.build() + logging.info("Python 构建、准备和打包完成。") except Exception as e: - logging.error(f"MinGW 构建过程中发生错误: {str(e)}") + logging.exception(f"构建过程中发生错误: {str(e)}") if __name__ == "__main__": diff --git a/.cid/darwin_python_builder.py b/.cid/darwin_python_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..56c95455041e9b43fc1a424eac5f45e766ae5f60 --- /dev/null +++ b/.cid/darwin_python_builder.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from pathlib import Path +from typing import List, Mapping +from python_builder import PythonBuilder, run_command +import platform + +class DarwinPythonBuilder(PythonBuilder): + def __init__(self, build_config) -> None: + super().__init__(build_config) + self._patch_ignore_file = self._source_dir / 'darwin_ignorefile.txt' + self.platform_arch = platform.machine() + + + @property + def _cc(self) -> Path: + return Path('/usr/bin/clang') + + + @property + def _cxx(self) -> Path: + return Path('/usr/bin/clang++') + + @property + def _strip(self) -> Path: + return Path('/usr/bin/strip') + + @property + def _cxxflags(self) -> List[str]: + return self._cflags.copy() + + @property + def _cflags(self) -> List[str]: + # macOS特有的编译标志（如最小系统版本、框架路径） + return [ + '-mmacosx-version-min=10.14', + '-DMACOSX_DEPLOYMENT_TARGET=10.14', + f'-I{str(self._deps_dir / "ffi" / "include")}', + ] + + @property + def _ldflags(self) -> List[str]: + # macOS特有的链接标志（如框架搜索路径、版本兼容） + return [ + #f'-L{str(self._deps_dir / "ffi" / "lib")}', + f'-L{str(self._deps_dir / "ssl" / "lib")}', + '-Wl,-rpath,@loader_path/../lib', + '-O2', '-fPIC', '-fstack-protector-strong', '-Wno-unused-command-line-argument' + ] + + @property + def _rcflags(self) -> List[str]: + # macOS无windres，此方法可留空或根据需要调整（如资源文件处理） + return [] + + def _deps_build(self) -> None: + """依赖构建逻辑调整为macOS环境（如使用clang编译）""" + self._logger.info("Starting Darwin dependency build process...") + + env = os.environ.copy() + env.update({ + 'CC': "/usr/bin/clang", # 使用Xcode的Clang + 'CXX': "/usr/bin/clang++", + 'CFLAGS': f"-arch {self.platform_arch} -mmacosx-version-min=10.15 -DHAVE_DYLD_SHARED_CACHE_CONTAINS_PATH=1", # 架构和系统版本 + 'LDFLAGS': f"-arch {self.platform_arch} -mmacosx-version-min=10.15" + }) + self._deps_build_openssl(env) + + + def _deps_build_libffi(self, env: dict) -> None: + """构建libffi库""" + self._logger.info("Building libffi...") + libffi_inner_dir = self._extract_libffi() + configure_cmd = [ + "./configure", + f"--prefix={self._deps_dir / 'ffi'}", + ] + run_command(configure_cmd, env=env, cwd=libffi_inner_dir) + run_command(['make', '-j16'], env=env, cwd=libffi_inner_dir) + run_command(['make', 'install'], env=env, cwd=libffi_inner_dir) + + + def _deps_build_openssl(self, env: dict) -> None: + """构建openssl库""" + self._logger.info("Building openssl...") + openssl_dir = self.repo_root / 'third_party' / 'openssl' + configure_cmd = [ + "./Configure", + f"--prefix={self._deps_dir / 'ssl'}", + "--shared", + ] + run_command(configure_cmd, env=env, cwd=openssl_dir) + run_command(['make', '-j16'], env=env, cwd=openssl_dir) + run_command(['make', 'install_sw'], env=env, cwd=openssl_dir) + + def _configure(self) -> None: + """配置参数调整为macOS目标""" + self._logger.info("Starting Darwin configuration...") + config_flags = [ + f'--prefix={self._install_dir}', + f'--with-openssl={self._deps_dir / "ssl"}', + '--enable-shared', + '--with-ensurepip=upgrade', + '--enable-loadable-sqlite-extensions', + '--disable-ipv6' + ] + cmd = [str(self._source_dir / 'configure')] + config_flags + cmd.append('CFLAGS={}'.format(' '.join(self._cflags))) + cmd.append('LDFLAGS={}'.format(' '.join(self._cflags + self._ldflags))) + run_command(cmd, env=self._env, cwd=self._build_dir) + + def _post_build(self) -> None: + self._logger.info("Starting Darwin Python post-build...") + super()._prepare_package() + self._dylib_rpath_setting() + super()._package() + + + def _copy_external_libs(self) -> None: + self._logger.info("Darwin Copy external_libs...") + # 定义源文件路径 + _external_libs = [self._deps_dir / 'ssl' / 'lib' / 'libssl.dylib', self._deps_dir / 'ssl' / 'lib' / 'libcrypto.dylib'] + # 定义目标目录 + target_dir = self._install_dir / 'lib' / 'python3.11' / 'lib' + # 创建目标目录（如果不存在） + target_dir.mkdir(parents=True, exist_ok=True) + + try: + for lib in _external_libs : + # 调用提取的方法拷贝 libffi-8.dll + self._copy_file_or_symlink_target(lib, target_dir) + except Exception as e: + self._logger.exception(f"Error copying external libraries: {e}") + + + def _dylib_rpath_setting(self) -> None: + self._logger.info("Darwin dylib rpath setting...") + prefix = str(self._deps_dir / 'ssl' / 'lib') + python_exec = self._install_dir / 'bin' / 'python3.11' + python_dylib = self._install_dir / 'lib' / 'libpython3.11.dylib' + dylib_dir = self._install_dir / 'lib' / 'python3.11' / 'lib' + _ssl_module_so = self._install_dir / 'lib' / 'python3.11' / 'lib-dynload' / '_ssl.cpython-311-darwin.so' + + try: + run_command(["install_name_tool", "-id", f"@rpath/{python_exec.name}", f"{python_exec}"]) + run_command(["install_name_tool", "-change", f"{python_dylib}", f"@rpath/{python_dylib.name}", f"{python_exec}"]) + run_command(["install_name_tool", "-id", f"@rpath/{python_dylib.name}", f"{python_dylib}"]) + + for dylib in dylib_dir.iterdir() : + try: + file_name = dylib.name + prev_install_name = f'{prefix}/{file_name}' + run_command(["install_name_tool", "-id", f"@rpath/{file_name}", f"{dylib}"]) + if 'ssl' in file_name: + run_command(["install_name_tool", "-change", f"{prefix}/libcrypto.3.dylib", "@rpath/libcrypto.3.dylib", f"{dylib}"]) + run_command(["install_name_tool", "-change", f"{prev_install_name}", f"@rpath/{file_name}", f"{_ssl_module_so}"]) + run_command(["codesign", "--remove-signature", f"{dylib}"]) + run_command(["codesign", "-f", "-s", "-", f"{dylib}"]) + except Exception as e: + # 记录当前dylib处理失败的错误，但不退出循环 + self._logger.exception(f"处理{dylib.name}时出错: {e}") + # 可以选择继续处理下一个或根据需要添加其他逻辑 + continue + except Exception as e: + self._logger.exception(f"Error dylib rpath setting: {e}") + raise \ No newline at end of file diff --git a/.cid/linux_python_builder.py b/.cid/linux_python_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..2493605eb2cba1863ed59859b0f9e0b3ae2f7914 --- /dev/null +++ b/.cid/linux_python_builder.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from pathlib import Path +import subprocess +from typing import List +from python_builder import PythonBuilder, run_command + +class LinuxPythonBuilder(PythonBuilder): + def __init__(self, build_config) -> None: + super().__init__(build_config) + # Linux平台特有的补丁文件 + # Linux工具链路径（示例使用系统GCC） + self._gcc_toolchain_dir = Path('/usr/bin').resolve() # 系统GCC路径 + # 构建目录定义 + # 补丁检测文件（平台特定命名） + self._patch_ignore_file = self._source_dir / 'linux_ignorefile.txt' + + # 验证Linux必要目录（示例检查源码目录和GCC） + if not self._source_dir.is_dir(): + raise ValueError(f'No such source directory "{self._source_dir}"') + if not (self._gcc_toolchain_dir / 'gcc').exists(): + raise ValueError(f'GCC not found in "{self._gcc_toolchain_dir}"') + + + @property + def _cflags(self) -> List[str]: + # Linux通用编译标志（如优化选项、调试符号） + return [ + '-D_FORTIFY_SOURCE=2 -O2', # 优化等级 + '-fPIC', + '-fstack-protector-strong', + '-Wno-unused-command-line-argument', + '-s' + ] + + @property + def _ldflags(self) -> List[str]: + # Linux通用链接标志（如动态库路径） + return [ + #f'-L{str(self._deps_dir / "ssl" / "lib64")}', + '-Wl,-rpath,\\$$ORIGIN/../lib', + '-Wl,--as-needed', + '-Wl,-z,relro', + '-Wl,-z,now' + ] + + @property + def _rcflags(self) -> List[str]: + # Linux无资源编译器，此方法可留空 + return [] + + def _deps_build(self) -> None: + """依赖构建逻辑调整为Linux环境（使用系统GCC）""" + self._logger.info("Starting Linux dependency build process...") + env = os.environ.copy() + env.update({ + 'CC': "/usr/bin/gcc", # 系统GCC + 'CXX': "/usr/bin/g++", + 'CFLAGS': "-O2 -fPIC -fstack-protector-strong -Wno-unused-command-line-argument", # 编译标志 + 'LDFLAGS': "-Wl,--as-needed -Wl,-z,relro,-z,now" # 链接优化 + }) + ## TODO: 使用openssl+rpath实现依赖可追溯 + #self._deps_build_openssl(env) + + + def _deps_build_libffi(self, env: dict) -> None: + """构建libffi库""" + self._logger.info("Building libffi...") + libffi_inner_dir = self._extract_libffi() + configure_cmd = [ + "./configure", + f"--prefix={self._deps_dir / 'ffi'}", + "--enable-shared", + "--build=x86_64-linux-gnu", # 构建平台 + "--host=x86_64-linux-gnu", # 目标平台 + "--disable-docs" + ] + run_command(configure_cmd, env=env, cwd=libffi_inner_dir) + run_command(['make', '-j16'], env=env, cwd=libffi_inner_dir) + run_command(['make', 'install'], env=env, cwd=libffi_inner_dir) + + + def _deps_build_openssl(self, env: dict) -> None: + """构建openssl库""" + self._logger.info("Building openssl...") + openssl_dir = self.repo_root / 'third_party' / 'openssl' + configure_cmd = [ + "./Configure", + f"--prefix={self._deps_dir / 'ssl'}", + "--shared", + ] + run_command(configure_cmd, env=env, cwd=openssl_dir) + run_command(['make', '-j16'], env=env, cwd=openssl_dir) + run_command(['make', 'install_sw'], env=env, cwd=openssl_dir) + + + def _configure(self) -> None: + """配置参数调整为Linux目标""" + self._logger.info("Starting Linux configuration...") + config_flags = [ + f'--prefix={self._install_dir}', + #f'--with-openssl={self._deps_dir / "ssl"}', + #f'--with-openssl-rpath={self._deps_dir / "ssl" / "lib64"}', + '--enable-shared', + '--with-ensurepip=upgrade', + '--disable-ipv6' + ] + cmd = [str(self._source_dir / 'configure')] + config_flags + cmd.append('CFLAGS={}'.format(' '.join(self._cflags))) + cmd.append('LDFLAGS={}'.format(' '.join(self._cflags + self._ldflags))) + run_command(cmd, env=self._env, cwd=self._build_dir) + + + def _post_build(self) -> None: + self._logger.info("Starting Linux Python post-build...") + super()._prepare_package() + glibc_version = self._get_glibc_version() + super()._package(glibc_version) + + + + def _copy_external_libs(self) -> None: + self._logger.info("Linux Copy external_libs...") + # 定义源文件路径 + #_external_libs = [self._deps_dir / 'ssl' / 'lib64' / 'libssl.so', self._deps_dir / 'ssl' / 'lib64' / 'libcrypto.so'] + # 定义目标目录 + #target_dir = self._install_dir / 'lib' / 'python3.11' / 'lib' + # 创建目标目录（如果不存在） + #target_dir.mkdir(parents=True, exist_ok=True) + + #try: + # for lib in _external_libs : + # # 调用提取的方法拷贝 libffi-8.dll + # self._copy_file_or_symlink_target(lib, target_dir) + #except Exception as e: + # self._logger.error(f"Error copying external libraries: {e}") + + + def _get_glibc_version(self): + """ + 通过执行 getconf 和 grep 命令获取 glibc 版本 + + 返回: + str: 成功则返回版本号（如 "2.35"），失败则返回 None + """ + try: + # 构造命令管道：getconf | grep + # 使用 shell=True 来支持管道操作 + result = subprocess.run( + 'getconf GNU_LIBC_VERSION | grep -oE \'[0-9]+\.[0-9]{2}\'', + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # 检查命令执行是否成功 + if result.returncode != 0: + self._logger.error(f"Command failed with error: {result.stderr.strip()}") + return None + + # 提取并清理版本号 + version = result.stdout.strip() + if version: + self._logger.info(f"Detected glibc version: {version}") + return version + else: + self._logger.warning("No version information found in command output") + return None + + except Exception as e: + self._logger.error(f"Error executing command: {str(e)}") + return None diff --git a/.cid/mingw_python_builder.py b/.cid/mingw_python_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..d1868de513cc27fba15d6b002b3547296be594e6 --- /dev/null +++ b/.cid/mingw_python_builder.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import os +from pathlib import Path +import subprocess +from typing import List, Mapping +from python_builder import PythonBuilder, run_command + +class MinGWPythonBuilder(PythonBuilder): + def __init__(self, build_config) -> None: + super().__init__(build_config) + + self.target_platform = "x86_64-w64-mingw32" + self._clang_toolchain_dir = Path( + os.path.join(self._prebuilts_path, 'mingw-w64', 'ohos', 'linux-x86_64', 'clang-mingw')).resolve() + self._mingw_install_dir = self._clang_toolchain_dir / self.target_platform + for directory in (self._mingw_install_dir, self._source_dir): + if not directory.is_dir(): + raise ValueError(f'No such directory "{directory}"') + + @property + def _env(self) -> Mapping[str, str]: + env = os.environ.copy() + toolchain_bin_dir = self._clang_toolchain_dir / 'bin' + env.update({ + 'CC': str(self._cc), + 'CXX': str(self._cxx), + 'WINDRES': str(toolchain_bin_dir / 'llvm-windres'), + 'AR': str(toolchain_bin_dir / 'llvm-ar'), + 'READELF': str(toolchain_bin_dir / 'llvm-readelf'), + 'LD': str(toolchain_bin_dir / 'ld.lld'), + 'DLLTOOL': str(toolchain_bin_dir / 'llvm-dlltoo'), + 'RANLIB': str(toolchain_bin_dir / 'llvm-ranlib'), + 'STRIP': str(self._strip), + 'CFLAGS': ' '.join(self._cflags), + 'CXXFLAGS': ' '.join(self._cxxflags), + 'LDFLAGS': ' '.join(self._ldflags), + 'RCFLAGS': ' '.join(self._rcflags), + 'CPPFLAGS': ' '.join(self._cflags), + 'LIBS': '-lffi -lssl -lcrypto' + }) + return env + + + @property + def _cc(self) -> Path: + return self._clang_toolchain_dir / 'bin' / 'clang' + + @property + def _cxx(self) -> Path: + return self._clang_toolchain_dir / 'bin' / 'clang++' + + @property + def _strip(self) -> Path: + return self._clang_toolchain_dir / 'bin' / 'llvm-strip' + + + @property + def _cflags(self) -> List[str]: + cflags = [ + f'-target {self.target_platform}', + f'--sysroot={self._mingw_install_dir}', + f'-fstack-protector-strong', + f'-I{str(self._deps_dir / "ffi" / "include")}', + f'-nostdinc', + f'-I{str(self._mingw_install_dir / "include")}', + f'-I{str(self._clang_toolchain_dir / "lib" / "clang" / "15.0.4" / "include")}' + ] + return cflags + + @property + def _ldflags(self) -> List[str]: + ldflags = [ + f'--sysroot={self._mingw_install_dir}', + f'-rtlib=compiler-rt', + f'-target {self.target_platform}', + f'-lucrt', + f'-lucrtbase', + f'-fuse-ld=lld', + f'-L{str(self._deps_dir / "ffi" / "lib")}', + ] + return ldflags + + @property + def _rcflags(self) -> List[str]: + return [f'-I{self._mingw_install_dir}/include'] + + def _deps_build(self) -> None: + self._logger.info("Starting MinGW dependency build process...") + # 调用提取的方法 + libffi_inner_dir = self._extract_libffi() + + env = os.environ.copy() + env.update({ + 'CC': "/bin/x86_64-w64-mingw32-gcc", + 'CXX': "/bin/x86_64-w64-mingw32-g++", + 'WINDRES': "/bin/x86_64-w64-mingw32-windres", + 'AR': "/bin/x86_64-w64-mingw32-ar", + 'READELF': "/bin/x86_64-w64-mingw32-readelf", + 'LD': "/bin/x86_64-w64-mingw32-ld", + 'DLLTOOL': "/bin/x86_64-w64-mingw32-dlltool", + 'RANLIB': "/bin/x86_64-w64-mingw32-gcc-ranlib", + 'STRIP': "/bin/x86_64-w64-mingw32-strip", + 'CFLAGS': "--sysroot=/usr/x86_64-w64-mingw32 -fstack-protector-strong", + 'CXXFLAGS': "--sysroot=/usr/x86_64-w64-mingw32 -fstack-protector-strong", + 'LDFLAGS': "--sysroot=/usr/x86_64-w64-mingw32", + 'RCFLAGS': "-I/usr/x86_64-w64-mingw32/include", + 'CPPFLAGS': "--sysroot=/usr/x86_64-w64-mingw32 -fstack-protector-strong" + }) + + configure_cmd = [ + "./configure", + f"--prefix={self._deps_dir / 'ffi'}", + "--enable-shared", + "--build=x86_64-pc-linux-gnu", + "--host=x86_64-w64-mingw32", + "--disable-symvers", + "--disable-docs" + ] + run_command(configure_cmd, env=env, cwd=libffi_inner_dir) + + # 执行 make -j16 + make_cmd = ['make', '-j16'] + run_command(make_cmd, env=env, cwd=libffi_inner_dir) + + # 执行 make install + make_install_cmd = ['make', 'install'] + run_command(make_install_cmd, env=env, cwd=libffi_inner_dir) + + def _configure(self) -> None: + self._logger.info("Starting MinGW configuration...") + run_command(['autoreconf', '-vfi'], cwd=self._source_dir) + build_platform = subprocess.check_output( + ['./config.guess'], cwd=self._source_dir).decode().strip() + config_flags = [ + f'--prefix={self._install_dir}', + f'--build={build_platform}', + f'--host={self.target_platform}', + f'--with-build-python={self._prebuilts_python_path}', + '--enable-shared', + '--without-ensurepip', + '--enable-loadable-sqlite-extensions', + '--disable-ipv6', + '--with-system-ffi' + ] + cmd = [str(self._source_dir / 'configure')] + config_flags + run_command(cmd, env=self._env, cwd=self._build_dir) + + + def _copy_external_libs(self) -> None: + self._logger.info("Copying external libraries...") + # 定义源文件路径 + _external_libs = [self._deps_dir / 'ffi' / 'bin' / 'libffi-8.dll', self._clang_toolchain_dir / self.target_platform / 'bin' / 'libssp-0.dll'] + # 定义目标目录 + target_dir = self._install_dir / 'lib' / 'python3.11' / 'lib-dynload' + # 创建目标目录（如果不存在） + target_dir.mkdir(parents=True, exist_ok=True) + + try: + for lib in _external_libs : + # 调用提取的方法拷贝 libffi-8.dll + self._copy_file_if_exists(lib, target_dir) + except Exception as e: + self._logger.error(f"Error copying external libraries: {e}") + + def _post_build(self) -> None: + self._logger.info("Starting MinGW Python post-build...") + super()._prepare_package() + super()._package() + + + def _clean_bin_dir(self) -> None: + self._logger.info("Cleaning MinGW bin directory...") + python_bin_dir = self._install_dir / 'bin' + if not python_bin_dir.is_dir(): + return + + windows_suffixes = ('.exe', '.dll') + for f in python_bin_dir.iterdir(): + if f.suffix not in windows_suffixes or f.is_symlink(): + self._logger.info(f"Removing file: {f}") + f.unlink() + continue + self._strip_in_place(f) diff --git a/.cid/patches/cpython_mingw_v3.11.4_20250509.patch b/.cid/patches/mingw/cpython_mingw_v3.11.4_20250509.patch similarity index 100% rename from .cid/patches/cpython_mingw_v3.11.4_20250509.patch rename to .cid/patches/mingw/cpython_mingw_v3.11.4_20250509.patch diff --git a/.cid/python_builder.py b/.cid/python_builder.py index c7bd92da85880d7d5ec7b9f3e8b2b4542de9501b..77ebca114c6321fb80ce7bbb7c6e2180dc931e3e 100644 --- a/.cid/python_builder.py +++ b/.cid/python_builder.py @@ -1,468 +1,495 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright (c) 2025 Huawei Device Co., Ltd. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os -from pathlib import Path -import shutil -import subprocess -from typing import List, Mapping -import binascii -import glob -import tarfile - - -# 配置日志记录 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler() - ] -) - - -def run_command(cmd, cwd=None, env=None): - logger = logging.getLogger(__name__) - try: - logger.info(f"Command: {' '.join(cmd)}") - result = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True, check=True) - if result.stdout: - logger.info(f"Command output: {result.stdout.strip()}") - if result.stderr: - logger.warning(f"Command error output: {result.stderr.strip()}") - except subprocess.CalledProcessError as e: - logger.error(f"Command failed: {' '.join(cmd)}. Error: {e.stderr.strip()}") - raise - -class BuildConfig: - def __init__(self, args): - self.REPOROOT_DIR = args.repo_root - self.OUT_PATH = args.out_path - self.LLDB_PY_VERSION = args.lldb_py_version - self.LLDB_PY_DETAILED_VERSION = args.lldb_py_detailed_version - self.MINGW_TRIPLE = args.mingw_triple - - -class PythonBuilder: - target_platform = "" - patches = [] - - - def __init__(self, build_config) -> None: - self.build_config = build_config - self.repo_root = Path(build_config.REPOROOT_DIR).resolve() - self._out_dir = Path(build_config.OUT_PATH).resolve() - self._lldb_py_version = build_config.LLDB_PY_VERSION - self._version = build_config.LLDB_PY_DETAILED_VERSION - version_parts = self._version.split('.') - self._major_version = version_parts[0] - self._source_dir = self.repo_root / 'third_party' / 'python' - self._patch_dir = self._source_dir / '.cid' / 'patches' - self._prebuilts_path = os.path.join(self.repo_root, 'prebuilts') - self._prebuilts_python_path = os.path.join(self._prebuilts_path, 'python', 'linux-x86', self._lldb_py_version, 'bin', - f'python{self._major_version}') - self._install_dir = "" - self._clean_patches() - logging.getLogger(__name__).addHandler(logging.FileHandler(self._out_dir / 'build.log')) - - @property - def _logger(self) -> logging.Logger: - return logging.getLogger(__name__) - - @property - def _cc(self) -> Path: - return self._clang_toolchain_dir / 'bin' / 'clang' - - @property - def _cflags(self) -> List[str]: - return [] - - @property - def _ldflags(self) -> List[str]: - return [] - - @property - def _cxx(self) -> Path: - return self._clang_toolchain_dir / 'bin' / 'clang++' - - @property - def _strip(self) -> Path: - return self._clang_toolchain_dir / 'bin' / 'llvm-strip' - - @property - def _cxxflags(self) -> List[str]: - return self._cflags.copy() - - @property - def _rcflags(self) -> List[str]: - return [] - - @property - def _env(self) -> Mapping[str, str]: - env = os.environ.copy() - clang_bin_dir = self._clang_toolchain_dir / 'bin' - - env.update({ - 'CC': str(self._cc), - 'CXX': str(self._cxx), - 'WINDRES': str(clang_bin_dir / 'llvm-windres'), - 'AR': str(clang_bin_dir / 'llvm-ar'), - 'READELF': str(clang_bin_dir / 'llvm-readelf'), - 'LD': str(clang_bin_dir / 'ld.lld'), - 'DLLTOOL': str(clang_bin_dir / 'llvm-dlltoo'), - 'RANLIB': str(clang_bin_dir / 'llvm-ranlib'), - 'STRIP': str(self._strip), - 'CFLAGS': ' '.join(self._cflags), - 'CXXFLAGS': ' '.join(self._cxxflags), - 'LDFLAGS': ' '.join(self._ldflags), - 'RCFLAGS': ' '.join(self._rcflags), - 'CPPFLAGS': ' '.join(self._cflags), - 'LIBS': '-lffi' - }) - return env - - def _configure(self) -> None: - self._logger.info("Starting configuration...") - return - - def _clean_patches(self) -> None: - self._logger.info("Cleaning patches...") - run_command(['git', 'reset', '--hard', 'HEAD'], cwd=self._source_dir) - run_command(['git', 'clean', '-df', '--exclude=.cid'], cwd=self._source_dir) - - def _pre_build(self) -> None: - self._deps_build() - self._apply_patches() - - def _apply_patches(self) -> None: - if hasattr(self, '_patch_ignore_file') and self._patch_ignore_file.is_file(): - self._logger.warning('Patches for Python have being applied, skip patching') - return - - if not self._patch_dir.is_dir(): - self._logger.warning('Patches are not found, skip patching') - return - - for patch in self._patch_dir.iterdir(): - if patch.is_file() and patch.name in self.patches: - cmd = ['git', 'apply', str(patch)] - self._logger.info(f"Applying patch: {patch.name}") - run_command(cmd, cwd=self._source_dir) - - - def _deps_build(self) -> None: - self._logger.info("Starting dependency build process...") - return - - def build(self) -> None: - self._logger.info("Starting build process...") - self._pre_build() - if hasattr(self, '_build_dir') and self._build_dir.exists(): - self._logger.info(f"Removing existing build directory: {self._build_dir}") - shutil.rmtree(self._build_dir) - if isinstance(self._install_dir, Path) and self._install_dir.exists(): - self._logger.info(f"Removing existing install directory: {self._install_dir}") - shutil.rmtree(self._install_dir) - if hasattr(self, '_build_dir'): - self._build_dir.mkdir(parents=True) - if isinstance(self._install_dir, Path): - self._install_dir.mkdir(parents=True) - self._configure() - self._install() - - def _install(self) -> None: - self._logger.info("Starting installation...") - num_jobs = os.cpu_count() or 8 - cmd = ['make', f'-j{num_jobs}', 'install'] - run_command(cmd, cwd=self._build_dir) - - def _strip_in_place(self, file: Path) -> None: - self._logger.info(f"Stripping file: {file}") - cmd = [ - str(self._strip), - str(file), - ] - run_command(cmd) - - def _clean_bin_dir(self) -> None: - self._logger.info("Cleaning bin directory...") - python_bin_dir = self._install_dir / 'bin' - if not python_bin_dir.is_dir(): - return - - windows_suffixes = ('.exe', '.dll') - for f in python_bin_dir.iterdir(): - if f.suffix not in windows_suffixes or f.is_symlink(): - self._logger.info(f"Removing file: {f}") - f.unlink() - continue - self._strip_in_place(f) - - def _remove_dir(self, dir_path: Path) -> None: - if dir_path.is_dir(): - self._logger.info(f"Removing directory: {dir_path}") - shutil.rmtree(dir_path) - - def _clean_share_dir(self) -> None: - self._logger.info("Cleaning share directory...") - share_dir = self._install_dir / 'share' - self._remove_dir(share_dir) - - def _clean_lib_dir(self) -> None: - self._logger.info("Cleaning lib directory...") - python_lib_dir = self._install_dir / 'lib' - pkgconfig_dir = python_lib_dir / 'pkgconfig' - self._remove_dir(pkgconfig_dir) - - def _remove_exclude(self) -> None: - self._logger.info("Removing excluded files and directories...") - exclude_dirs_tuple = ( - f'config-{self._major_version}', - '__pycache__', - 'idlelib', - 'tkinter', 'turtledemo', - 'test', 'tests' - ) - exclude_files_tuple = ( - 'bdist_wininst.py', - 'turtle.py', - '.whl', - '.pyc', '.pickle' - ) - - for root, dirs, files in os.walk(self._install_dir / 'lib'): - for item in dirs: - if item.startswith(exclude_dirs_tuple): - self._logger.info(f"Removing directory: {os.path.join(root, item)}") - shutil.rmtree(os.path.join(root, item)) - for item in files: - if item.endswith(exclude_files_tuple): - self._logger.info(f"Removing file: {os.path.join(root, item)}") - os.remove(os.path.join(root, item)) - - def _copy_external_libs(self) -> None: - self._logger.info("Copying external libraries...") - # 定义源文件路径 - _external_libs = [self._deps_dir / 'ffi' / 'bin' / 'libffi-8.dll', self._clang_toolchain_dir / self.build_config.MINGW_TRIPLE / 'bin' / 'libssp-0.dll'] - # 定义目标目录 - target_dir = self._install_dir / 'lib' / 'python3.11' / 'lib-dynload' - # 创建目标目录（如果不存在） - target_dir.mkdir(parents=True, exist_ok=True) - - try: - for lib in _external_libs : - # 调用提取的方法拷贝 libffi-8.dll - self._copy_file_if_exists(lib, target_dir) - except Exception as e: - self._logger.error(f"Error copying external libraries: {e}") - - def _copy_file_if_exists(self, src_path: Path, dest_dir: Path) -> None: - """ - 若源文件存在，则将其拷贝到目标目录，并记录相应日志；若不存在，则记录警告日志。 - - :param src_path: 源文件的路径 - :param dest_dir: 目标目录的路径 - """ - if src_path.exists(): - shutil.copy2(src_path, dest_dir) - self._logger.info(f"Copied {src_path} to {dest_dir}") - else: - self._logger.warning(f"{src_path} does not exist. Skipping.") - - def _is_elf_file(self, file_path: Path) -> bool: - with open(file_path, 'rb') as f: - magic_numbers = f.read(4) - hex_magic_number = binascii.hexlify(magic_numbers).decode('utf-8') - return hex_magic_number == '7f454c46' - - @property - def install_dir(self) -> str: - return str(self._install_dir) - - -class MinGWPythonBuilder(PythonBuilder): - def __init__(self, build_config) -> None: - super().__init__(build_config) - - self.target_platform = "x86_64-w64-mingw32" - self.patches = [f'cpython_mingw_v{self._version}.patch'] - self._clang_toolchain_dir = Path( - os.path.join(self._prebuilts_path, 'mingw-w64', 'ohos', 'linux-x86_64', 'clang-mingw')).resolve() - self._mingw_install_dir = self._clang_toolchain_dir / build_config.MINGW_TRIPLE - self._build_dir = self._out_dir / 'python-windows-build' - self._install_dir = self._out_dir / 'python-windows-install' - self._deps_dir = self._out_dir / 'python-windows-deps' - # This file is used to detect whether patches are applied - self._patch_ignore_file = self._source_dir / 'mingw_ignorefile.txt' - - for directory in (self._mingw_install_dir, self._source_dir): - if not directory.is_dir(): - raise ValueError(f'No such directory "{directory}"') - - def _extract_libffi(self): - """ - 定位 libffi-*.tar.gz 文件，清理输出目录后，将其直接解压到 out/libffi 目录。 - - Returns: - Path: 解压后的 libffi 内部目录的路径。 - - Raises: - FileNotFoundError: 若未找到 libffi-*.tar.gz 文件。 - Exception: 若无法获取 libffi 压缩包的内部目录。 - """ - # 找到 libffi-*.tar.gz 包 - libffi_tar_gz_files = glob.glob(str(self.repo_root / 'third_party' / 'libffi' / 'libffi-*.tar.gz')) - if not libffi_tar_gz_files: - self._logger.error("No libffi-*.tar.gz file found in third_party/libffi directory.") - raise FileNotFoundError("No libffi-*.tar.gz file found.") - libffi_tar_gz = libffi_tar_gz_files[0] - - # 清理 out/libffi 目录 - libffi_extract_dir = self._out_dir / 'libffi' - if libffi_extract_dir.exists(): - self._logger.info(f"Cleaning existing libffi directory: {libffi_extract_dir}") - shutil.rmtree(libffi_extract_dir) - libffi_extract_dir.mkdir(parents=True) - - # 直接解压 libffi-*.tar.gz 到 out/libffi 目录 - with tarfile.open(libffi_tar_gz, 'r:gz') as tar: - tar.extractall(path=libffi_extract_dir) - # 获取解压后的目录名 - members = tar.getmembers() - if members: - libffi_inner_dir = libffi_extract_dir / members[0].name - else: - self._logger.error("Failed to get inner directory of libffi tarball.") - raise Exception("Failed to get inner directory of libffi tarball.") - return libffi_inner_dir - - @property - def _cflags(self) -> List[str]: - cflags = [ - f'-target {self.target_platform}', - f'--sysroot={self._mingw_install_dir}', - f'-fstack-protector-strong', - f'-I{str(self._deps_dir / "ffi" / "include")}', - f'-nostdinc', - f'-I{str(self._mingw_install_dir / "include")}', - f'-I{str(self._clang_toolchain_dir / "lib" / "clang" / "15.0.4" / "include")}' - ] - return cflags - - @property - def _ldflags(self) -> List[str]: - ldflags = [ - f'--sysroot={self._mingw_install_dir}', - f'-rtlib=compiler-rt', - f'-target {self.target_platform}', - f'-lucrt', - f'-lucrtbase', - f'-fuse-ld=lld', - f'-L{str(self._deps_dir / "ffi" / "lib")}', - ] - return ldflags - - @property - def _rcflags(self) -> List[str]: - return [f'-I{self._mingw_install_dir}/include'] - - def _deps_build(self) -> None: - self._logger.info("Starting MinGW dependency build process...") - # 调用提取的方法 - libffi_inner_dir = self._extract_libffi() - - env = os.environ.copy() - env.update({ - 'CC': "/bin/x86_64-w64-mingw32-gcc", - 'CXX': "/bin/x86_64-w64-mingw32-g++", - 'WINDRES': "/bin/x86_64-w64-mingw32-windres", - 'AR': "/bin/x86_64-w64-mingw32-ar", - 'READELF': "/bin/x86_64-w64-mingw32-readelf", - 'LD': "/bin/x86_64-w64-mingw32-ld", - 'DLLTOOL': "/bin/x86_64-w64-mingw32-dlltool", - 'RANLIB': "/bin/x86_64-w64-mingw32-gcc-ranlib", - 'STRIP': "/bin/x86_64-w64-mingw32-strip", - 'CFLAGS': "--sysroot=/usr/x86_64-w64-mingw32 -fstack-protector-strong", - 'CXXFLAGS': "--sysroot=/usr/x86_64-w64-mingw32 -fstack-protector-strong", - 'LDFLAGS': "--sysroot=/usr/x86_64-w64-mingw32", - 'RCFLAGS': "-I/usr/x86_64-w64-mingw32/include", - 'CPPFLAGS': "--sysroot=/usr/x86_64-w64-mingw32 -fstack-protector-strong" - }) - - configure_cmd = [ - "./configure", - f"--prefix={self._deps_dir / 'ffi'}", - "--enable-shared", - "--build=x86_64-pc-linux-gnu", - "--host=x86_64-w64-mingw32", - "--disable-symvers", - "--disable-docs" - ] - run_command(configure_cmd, env=env, cwd=libffi_inner_dir) - - # 执行 make -j16 - make_cmd = ['make', '-j16'] - run_command(make_cmd, env=env, cwd=libffi_inner_dir) - - # 执行 make install - make_install_cmd = ['make', 'install'] - run_command(make_install_cmd, env=env, cwd=libffi_inner_dir) - - def _configure(self) -> None: - self._logger.info("Starting MinGW configuration...") - run_command(['autoreconf', '-vfi'], cwd=self._source_dir) - build_platform = subprocess.check_output( - ['./config.guess'], cwd=self._source_dir).decode().strip() - config_flags = [ - f'--prefix={self._install_dir}', - f'--build={build_platform}', - f'--host={self.target_platform}', - f'--with-build-python={self._prebuilts_python_path}', - '--enable-shared', - '--without-ensurepip', - '--enable-loadable-sqlite-extensions', - '--disable-ipv6', - '--with-pydebug', - '--with-system-ffi' - ] - cmd = [str(self._source_dir / 'configure')] + config_flags - run_command(cmd, env=self._env, cwd=self._build_dir) - - def prepare_for_package(self) -> None: - self._logger.info("Preparing MinGW build for packaging...") - self._clean_bin_dir() - self._clean_share_dir() - self._clean_lib_dir() - self._remove_exclude() - self._copy_external_libs() - - def package(self) -> None: - self._logger.info("Packaging MinGW build...") - archive = self._out_dir / f'python-mingw-x86-{self._version}.tar.gz' - if archive.exists(): - self._logger.info(f"Removing existing archive: {archive}") - archive.unlink() - cmd = [ - 'tar', - '-czf', - str(archive), - '--exclude=__pycache__', - '--transform', - f's,^,python/windows-x86/{self._lldb_py_version}/,', - ] + [f.name for f in self._install_dir.iterdir()] - run_command(cmd, cwd=self._install_dir) +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +from pathlib import Path +import shutil +import subprocess +from typing import List, Mapping +import datetime +import platform +import tarfile +import glob +import re + + + +# 配置日志记录 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s', + handlers=[ + logging.StreamHandler() + ] +) + + +def run_command(cmd, cwd=None, env=None): + logger = logging.getLogger(__name__) + try: + logger.info(f"Command: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True, check=True) + if result.stdout: + logger.info(f"Command output: {result.stdout.strip()}") + if result.stderr: + logger.warning(f"Command error output: {result.stderr.strip()}") + except subprocess.CalledProcessError as e: + logger.error(f"Command failed: {' '.join(cmd)}. Error: {e.stderr.strip()}") + raise + +class BuildConfig: + def __init__(self, args): + self.REPOROOT_DIR = args.repo_root + self.OUT_PATH = args.out_path + self.LLDB_PY_VERSION = args.lldb_py_version + self.LLDB_PY_DETAILED_VERSION = f"{args.lldb_py_version}_{datetime.datetime.now().strftime('%Y%m%d')}" + self.TARGET_OS = args.target_os + self.TARGET_ARCH = args.target_arch + self.HOST_OS = platform.system().lower() + self.HOST_ARCH = 'x86' if platform.machine().startswith('x86') else 'arm64' + self.PACKAGE_NAME = "windows-x86" if args.target_os == "mingw" else f"{args.target_os}-{args.target_arch}" + +class PythonBuilder: + target_platform = "" + patches = [] + + + def __init__(self, build_config) -> None: + self.build_config = build_config + self.repo_root = Path(build_config.REPOROOT_DIR).resolve() + self._out_dir = Path(build_config.OUT_PATH).resolve() + self._source_dir = self.repo_root / 'third_party' / 'python' + self._patch_dir = self._source_dir / '.cid' / 'patches' / build_config.TARGET_OS + self._build_dir = self._out_dir / 'python-build' + self._deps_dir = self._out_dir / 'python-deps' + self._install_dir = self._out_dir / 'python-install' / build_config.PACKAGE_NAME / build_config.LLDB_PY_VERSION + self._lldb_py_version = build_config.LLDB_PY_VERSION + self._version = build_config.LLDB_PY_DETAILED_VERSION + version_parts = self._version.split('.') + prebuilt_python_sub_dir = f'{build_config.HOST_OS}-{build_config.HOST_ARCH}' + self._major_version = version_parts[0] + self._prebuilts_path = os.path.join(self.repo_root, 'prebuilts') + self._prebuilts_python_path = os.path.join(self._prebuilts_path, 'python', prebuilt_python_sub_dir, 'current', 'bin', + f'python{self._major_version}') + logging.getLogger(__name__).addHandler(logging.FileHandler(self._out_dir / 'build.log')) + + @property + def _logger(self) -> logging.Logger: + return logging.getLogger(__name__) + + @property + def _cc(self) -> Path: + return Path('/usr/bin/gcc') + + @property + def _cflags(self) -> List[str]: + return [] + + @property + def _ldflags(self) -> List[str]: + return [] + + @property + def _cxx(self) -> Path: + return Path('/usr/bin/g++') + + @property + def _strip(self) -> Path: + return Path('/usr/bin/strip') + + @property + def _cxxflags(self) -> List[str]: + return self._cflags.copy() + + @property + def _rcflags(self) -> List[str]: + return [] + + @property + def _env(self) -> Mapping[str, str]: + env = os.environ.copy() + env.update({ + 'CC': str(self._cc), + 'CXX': str(self._cxx), + 'STRIP': str(self._strip), + 'CFLAGS': ' '.join(self._cflags), + 'CXXFLAGS': ' '.join(self._cxxflags), + 'LDFLAGS': ' '.join(self._ldflags), + 'RCFLAGS': ' '.join(self._rcflags), + 'CPPFLAGS': ' '.join(self._cflags), + }) + return env + + + @property + def install_dir(self) -> str: + return str(self._install_dir) + + def build(self) -> None: + self._logger.info("Starting build process...") + self._pre_build() + self._build() + self._post_build() + + + def _pre_build(self) -> None: + self._deps_build() + if self.build_config.TARGET_OS == "mingw": + self._clean_patches() + self._apply_patches() + + + def _clean_patches(self) -> None: + self._logger.info("Cleaning patches...") + run_command(['git', 'reset', '--hard', 'HEAD'], cwd=self._source_dir) + run_command(['git', 'clean', '-df', '--exclude=.cid'], cwd=self._source_dir) + + + def _deps_build(self) -> None: + self._logger.info("Starting dependency build process...") + return + + + def _apply_patches(self) -> None: + if hasattr(self, '_patch_ignore_file') and self._patch_ignore_file.is_file(): + self._logger.warning('Patches for Python have being applied, skip patching') + return + + if not self._patch_dir.is_dir(): + self._logger.warning('Patches are not found, skip patching') + return + for patch in self._patch_dir.iterdir(): + if patch.is_file(): + cmd = ['git', 'apply', str(patch)] + self._logger.info(f"Applying patch: {patch.name}") + run_command(cmd, cwd=self._source_dir) + + + def _build(self): + self._prepare_build_dir() + self._configure() + self._install() + + + def _prepare_build_dir(self): + if hasattr(self, '_build_dir') and self._build_dir.exists(): + self._logger.info(f"Removing existing build directory: {self._build_dir}") + shutil.rmtree(self._build_dir) + if isinstance(self._install_dir, Path) and self._install_dir.exists(): + self._logger.info(f"Removing existing install directory: {self._install_dir}") + shutil.rmtree(self._install_dir) + if hasattr(self, '_build_dir'): + self._build_dir.mkdir(parents=True) + if isinstance(self._install_dir, Path): + self._install_dir.mkdir(parents=True) + + + def _configure(self) -> None: + self._logger.info("Starting configuration...") + return + + + def _install(self) -> None: + self._logger.info("Starting installation...") + num_jobs = os.cpu_count() or 8 + cmd = ['make', f'-j{num_jobs}', 'install'] + run_command(cmd, cwd=self._build_dir) + + + def _post_build(self) -> None: + self._logger.info("Starting post-build...") + self._prepare_package() + self._package("") + + + def _prepare_package(self) -> None: + self._logger.info("Preparing package...") + if self.build_config.TARGET_OS != "mingw": + self._modify_bin_file_shebang() + self._upgrade_pip_and_setuptools() + self._clean_bin_dir() + self._clean_share_dir() + self._clean_lib_dir() + self._copy_external_libs() + self._strip_libs() + + + + def _clean_bin_dir(self) -> None: + self._logger.info("ByPass Cleaning bin directory...") + + + def _strip_in_place(self, file: Path) -> None: + self._logger.info(f"Stripping file: {file}") + cmd = [ + str(self._strip), + '-x', + '-S', + str(file), + ] + run_command(cmd) + + + def _clean_share_dir(self) -> None: + self._logger.info("Cleaning share directory...") + share_dir = self._install_dir / 'share' + self._remove_dir(share_dir) + + + def _clean_lib_dir(self) -> None: + self._logger.info("Cleaning lib directory...") + python_lib_dir = self._install_dir / 'lib' + pkgconfig_dir = python_lib_dir / 'pkgconfig' + self._remove_dir(pkgconfig_dir) + + + def _remove_exclude(self) -> None: + self._logger.info("Removing excluded files and directories...") + exclude_dirs_tuple = ( + f'config-{self._major_version}', + '__pycache__', + 'idlelib', + 'tkinter', 'turtledemo', + 'test', 'tests' + ) + exclude_files_tuple = ( + 'bdist_wininst.py', + 'turtle.py', + '.whl', + '.pyc', '.pickle' + ) + + for root, dirs, files in os.walk(self._install_dir / 'lib'): + for item in dirs: + if item.startswith(exclude_dirs_tuple): + self._logger.info(f"Removing directory: {os.path.join(root, item)}") + shutil.rmtree(os.path.join(root, item)) + for item in files: + if item.endswith(exclude_files_tuple): + self._logger.info(f"Removing file: {os.path.join(root, item)}") + os.remove(os.path.join(root, item)) + + + def _strip_libs(self) -> None: + so_pattern = re.compile(r'.+\.so(\.\d+)*$') + directories = [ + self._install_dir / 'lib', + self._install_dir / 'lib' / 'python3.11' / 'lib-dynload' + ] + + for root_dir in directories: + if not root_dir.exists(): + self._logger.warning(f"Directory not found: {root_dir}") + continue + + for root, _, files in os.walk(root_dir): + for item in files: + if so_pattern.match(item): + file_path = os.path.join(root, item) + self._logger.info(f"Stripping file: {file_path}") + self._strip_in_place(file_path) + + + def _package(self, glibc_version="") -> None: + self._logger.info("Packaging build...") + platform = self.build_config.PACKAGE_NAME + if glibc_version: + package_name = f"python-{platform}-GLIBC{glibc_version}-{self._version}.tar.gz" + else: + package_name = f"python-{platform}-{self._version}.tar.gz" + archive = self._out_dir / package_name + package_dir = self._out_dir / 'python-install' + if archive.exists(): + self._logger.info(f"Removing existing archive: {archive}") + archive.unlink() + exclude_dirs = [ + "lib/python*/config-*", + "*.pyc", + "__pycache__", + "*.pickle", + "test", + "tests", + "tkinker", + "turtledemo", + "idlelib", + "turtle.py", + "wininst-*", + "bdist_wininst.py", + "*.whl" + ] + cmd = [ + 'tar', + '-czf', + str(archive), + ] + for p in exclude_dirs: + cmd.append("--exclude") + cmd.append(p) + cmd += [f.name for f in package_dir.iterdir()] + run_command(cmd, cwd=package_dir ) + + + def _copy_file_if_exists(self, src_path: Path, dest_dir: Path) -> None: + """ + 若源文件存在，则将其拷贝到目标目录，并记录相应日志；若不存在，则记录警告日志。 + + :param src_path: 源文件的路径 + :param dest_dir: 目标目录的路径 + """ + if src_path.exists(): + shutil.copy2(src_path, dest_dir) + self._logger.info(f"Copied {src_path} to {dest_dir}") + else: + self._logger.warning(f"{src_path} does not exist. Skipping.") + + + def _remove_dir(self, dir_path: Path) -> None: + if dir_path.is_dir(): + self._logger.info(f"Removing directory: {dir_path}") + shutil.rmtree(dir_path) + + + def _extract_libffi(self): + """ + 定位 libffi-*.tar.gz 文件，清理输出目录后，将其直接解压到 out/libffi 目录。 + + Returns: + Path: 解压后的 libffi 内部目录的路径。 + + Raises: + FileNotFoundError: 若未找到 libffi-*.tar.gz 文件。 + Exception: 若无法获取 libffi 压缩包的内部目录。 + """ + # 找到 libffi-*.tar.gz 包 + libffi_tar_gz_files = glob.glob(str(self.repo_root / 'third_party' / 'libffi' / 'libffi-*.tar.gz')) + if not libffi_tar_gz_files: + self._logger.error("No libffi-*.tar.gz file found in third_party/libffi directory.") + raise FileNotFoundError("No libffi-*.tar.gz file found.") + libffi_tar_gz = libffi_tar_gz_files[0] + + # 清理 out/libffi 目录 + libffi_extract_dir = self._out_dir / 'libffi' + if libffi_extract_dir.exists(): + self._logger.info(f"Cleaning existing libffi directory: {libffi_extract_dir}") + shutil.rmtree(libffi_extract_dir) + libffi_extract_dir.mkdir(parents=True) + + # 直接解压 libffi-*.tar.gz 到 out/libffi 目录 + with tarfile.open(libffi_tar_gz, 'r:gz') as tar: + tar.extractall(path=libffi_extract_dir) + # 获取解压后的目录名 + members = tar.getmembers() + if members: + libffi_inner_dir = libffi_extract_dir / members[0].name + else: + self._logger.error("Failed to get inner directory of libffi tarball.") + raise Exception("Failed to get inner directory of libffi tarball.") + return libffi_inner_dir + + + def _copy_file_or_symlink_target(self, source_path, dest_dir): + """ + 拷贝文件到目标目录： + - 若源路径是符号链接，则拷贝其指向的实际文件 + - 若源路径是普通文件，则直接拷贝该文件本身 + + 参数: + source_path: 源文件或符号链接的路径 + dest_dir: 拷贝操作的目标目录 + """ + # Check if source path exists + if not os.path.exists(source_path): + self._logger.error(f"Source path does not exist: {source_path}") + return + + # Determine the actual file path to copy + if os.path.islink(source_path): + # For symbolic links, get the target path + actual_path = os.path.realpath(source_path) + self._logger.info(f"Symbolic link detected, pointing to: {actual_path}") + else: + # For regular files, use the source path directly + actual_path = source_path + + # Ensure target directory exists + os.makedirs(dest_dir, exist_ok=True) + + # Perform the copy operation (preserving metadata) + shutil.copy2(actual_path, dest_dir) + self._logger.info(f"Successfully copied to {os.path.join(dest_dir, os.path.basename(actual_path))}") + + + def _modify_bin_file_shebang(self): + self._logger.info("Modify bin file shebang...") + python_bin_dir = self._install_dir / 'bin' + for file in python_bin_dir.iterdir(): + self._modify_file_shebang(file) + + + def _modify_file_shebang(self, file_path): + """ + 修改文件中的shebang行，仅校验第一行是否以#!开头 + + 参数: + file_path: 要修改的文件路径 + 返回: + bool: 修改成功返回True，否则返回False + """ + # 检查文件是否存在 + if not os.path.exists(file_path): + self._logger.error(f"File not found: {file_path}") + return False + + # 检查文件是否可读写 + if not os.access(file_path, os.R_OK | os.W_OK): + self._logger.error(f"No read/write permission for file: {file_path}") + return False + + try: + # 读取文件第一行 + with open(file_path, 'r') as f: + first_line = f.readline().rstrip('\n') # 移除换行符 + + # 只校验第一行是否以#!开头 + if not first_line.startswith('#!'): + self._logger.info(f"File does not have a shebang line: {file_path}") + return False + # 检查是否需要修改 + else: + # 读取所有内容并修改第一行 + with open(file_path, 'r') as f: + lines = f.readlines() + + lines[0] = '#!./python3.11\n' + + # 写回文件 + with open(file_path, 'w') as f: + f.writelines(lines) + + self._logger.info(f"Updated shebang in: {file_path}") + return True + + except Exception as e: + self._logger.warning(f"Error updating shebang in {file_path}: {e}") + + + def _upgrade_pip_and_setuptools(self): + self._logger.info("Upgrade pip and setuptools...") + pip_path = self._install_dir / 'bin' / 'pip3' + run_command([str(pip_path), 'install', '--upgrade', 'pip', '--index-url', 'https://mirrors.huaweicloud.com/repository/pypi/simple', '--trusted-host', 'mirrors.huaweicloud.com'], cwd=self._install_dir / 'bin') + run_command([str(pip_path), 'install', '--upgrade', 'setuptools', '--index-url', 'https://mirrors.huaweicloud.com/repository/pypi/simple', '--trusted-host', 'mirrors.huaweicloud.com'], cwd=self._install_dir / 'bin')