From be51dadecfda31feeb84e3b348be839d963e6340 Mon Sep 17 00:00:00 2001 From: chunlin Date: Mon, 28 Apr 2025 10:28:33 +0800 Subject: [PATCH 1/6] reconstruct install module --- src/deployService.py | 33 ++++++++ src/deploymentConfig.py | 25 ++++++ src/envFactory.py | 86 +++++++++++++++++++ src/installService.py | 182 ++++++---------------------------------- src/installTypes.py | 8 -- src/jsonService.py | 5 ++ src/pathBuilder.py | 68 ++++++++++----- src/softwareFactory.py | 9 +- src/softwareTypes.py | 3 + 9 files changed, 231 insertions(+), 188 deletions(-) create mode 100644 src/deployService.py create mode 100644 src/deploymentConfig.py create mode 100644 src/envFactory.py diff --git a/src/deployService.py b/src/deployService.py new file mode 100644 index 0000000..d8b6f35 --- /dev/null +++ b/src/deployService.py @@ -0,0 +1,33 @@ +from pathBuilder import PathFactory +from deploymentConfig import DeploymentConfig +from softwareTypes import SoftwareType, SoftwareProfile, EnvironmentProfile +from installTypes import InstallMode + +class DeploymentService: + def __init__(self, config: DeploymentConfig): + self.factory = PathFactory(config) + self.config = config + + def resolve_install_path( + self, + software: SoftwareProfile, + env: EnvironmentProfile + ) -> Path: + """生成规范化安装路径""" + self._validate_input(software, env) + + builder = self.factory.get_builder(software.software_type) + path = builder.build_path(software, env) + return path.absolute() + + def _validate_input(self, software: SoftwareProfile, env: EnvironmentProfile): + if software.software_type == SoftwareType.MPI and not env.mpi_version: + raise ValueError("MPI software requires MPI version in environment profile") + + if software.requires_mpi and not env.mpi_version: + raise ValueError("MPI-dependent software requires MPI version specification") + +if __name__ == '__main__': + dconfig = DeploymentConfig(InstallMode.NORMAL) + dservice = DeploymentService() + diff --git a/src/deploymentConfig.py b/src/deploymentConfig.py new file mode 100644 index 0000000..bf0c279 --- /dev/null +++ b/src/deploymentConfig.py @@ -0,0 +1,25 @@ +from pathlib import Path +from installTypes import InstallMode + +class DeploymentConfig: + def __init__(self, mode, paths): + self.is_pro = mode == InstallMode.PRO + # 基础路径配置 + self.mpi_base = Path(paths['MPI_PATH']) + self.compiler_base = Path(paths['COMPILER_PATH']) + self.lib_base = Path(paths['LIBS_PATH']) + self.util_base = Path(paths['UTILS_PATH']) + self.app_base = Path(paths['APP_PATH']) + self.misc_base = Path(paths['MISC_PATH']) + # 路径模式配置 + self.pro_path_pattern = "{name}{version}-{compiler}{compiler_ver}/{version}" + self.normal_path_pattern = "{name}/{version}-{compiler}{compiler_ver}" + + @property + def pro_mpi_path_pattern(self) -> Path: + return self.pro_path_pattern + + @property + def normal_mpi_path_pattern(self) -> Path: + return self.normal_path_pattern + diff --git a/src/envFactory.py b/src/envFactory.py new file mode 100644 index 0000000..ad1b045 --- /dev/null +++ b/src/envFactory.py @@ -0,0 +1,86 @@ +import os +from typing import Dict +from softwareTypes import SoftwareProfile, SoftwareType, EnvironmentProfile +from detectorService import GCCDetector, ClangDetector,NVCCDetector,ICCDetector +from detectorService import HMPIDetector, OpenMPIDetector, MPICHDetector +from versionParser import VersionParser + +class EnvFactory: + def __init__(self): + self.ROOT = os.getcwd() + self.gcc_detector = GCCDetector() + self.clang_detector = ClangDetector() + self.icc_detector = ICCDetector() + self.nvcc_detector = NVCCDetector() + + self.hmpi_detector = HMPIDetector() + self.openmpi_detector = OpenMPIDetector() + self.mpich_detector = MPICHDetector() + self._compilers = { + "GCC":self.gcc_detector.detect, + "CLANG":self.clang_detector.detect, + "NVC":self.nvcc_detector.detect, + "ICC":self.icc_detector.detect, + "BISHENG": self.clang_detector.detect + } + + def create_profile(self, software: SoftwareProfile, compiler_mpi_info: str) -> EnvironmentProfile: + """创建环境信息对象的统一入口""" + compiler_mpi_info = self._check_compiler_mpi(self._compilers.keys(), compiler_mpi_info) + stype = software.software_type + if stype != SoftwareType.LIB and stype != SoftwareType.MPI and stype != SoftwareType.APP: + return None + compiler_info = self._get_compiler_info(compiler_mpi_info) + profile = EnvironmentProfile( + compiler_name=compiler_info['cname'], + compiler_version=compiler_info['full_version'], + compiler_major_version=compiler_info['cmversion'] + ) + if software.use_mpi: + mpi_info = self._get_mpi_info() + profile.mpi_name = mpi_info['name'] + profile.mpi_version = mpi_info['full_version'] + profile.mpi_major_version = mpi_info['mversion'] + return profile + + def _check_compiler_mpi(self, compiler_list, compiler_mpi_info): + no_compiler = ["COM","ANY","MISC","APP"] + is_valid = False + compiler_mpi_info = compiler_mpi_info.upper() + valid_list = [] + for compiler in compiler_list: + valid_list.append(compiler) + valid_list.append(f'{compiler}+MPI') + valid_list += no_compiler + for valid_para in valid_list: + if compiler_mpi_info == valid_para: + is_valid = True + break + if not is_valid: + print(f"compiler or mpi info error, Only {valid_list.join('/').lower()} is supported") + sys.exit() + return compiler_mpi_info + + def _get_compiler_info(self, compiler_mpi_info): + compiler_info = None + for compiler, info_func in self._compilers.items(): + if compiler in compiler_mpi_info: + compiler_info = info_func() + break + if compiler_info == None: + print(f"The specified {compiler_mpi_info} Compiler not found!") + sys.exit() + cname = compiler_info['cname'] + cfullver = compiler_info['full_version'] + print(f"Use Compiler: {cname} {cfullver}") + return compiler_info + + def _get_mpi_info(self): + mpich_info = self.mpich_detector.detect() + if mpich_info:return mpich_info + hmpi_info = self.hmpi_detector.detect() + if hmpi_info:return hmpi_info + openmpi_info = self.openmpi_detector.detect() + if openmpi_info:return openmpi_info + print("MPI not found, please install MPI first.") + sys.exit() diff --git a/src/installService.py b/src/installService.py index 73503a5..a695db9 100644 --- a/src/installService.py +++ b/src/installService.py @@ -15,10 +15,11 @@ from jsonService import JSONService from installStrategy import PathStrategyFactory from installTypes import InstallMode from versionParser import VersionParser -from detectorService import GCCDetector, ClangDetector,NVCCDetector,ICCDetector -from detectorService import HMPIDetector, OpenMPIDetector, MPICHDetector from softwareFactory import SoftwareFactory +from envFactory import EnvFactory from softwareTypes import SoftwareType +from pathBuilder import PathFactory +from deploymentConfig import DeploymentConfig class Singleton(type): @@ -34,31 +35,27 @@ class Singleton(type): class InstallService(object,metaclass=Singleton): PACKAGE = 'package' FULL_VERSION='full_version' + INSTALL_JSON = "install.json" + LINK_FILE_NAME = 'linkpathtomodules.sh' def __init__(self): self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() + self.env_factory = EnvFactory() self.software_factory = SoftwareFactory() self.ROOT = os.getcwd() - self.gcc_detector = GCCDetector() - self.clang_detector = ClangDetector() - self.icc_detector = ICCDetector() - self.nvcc_detector = NVCCDetector() - - self.hmpi_detector = HMPIDetector() - self.openmpi_detector = OpenMPIDetector() - self.mpich_detector = MPICHDetector() - self._mode = self._detect_mode() self._init_paths() - self.INSTALL_INFO_PATH = os.path.join(self.SOFTWARE_PATH, "install.json") + self.deployment_config = DeploymentConfig(self._mode, self._paths) + self.path_factory = PathFactory(self.deployment_config) + self.INSTALL_INFO_PATH = os.path.join(self.SOFTWARE_PATH, self.INSTALL_JSON) self._is_first_install = self._set_first_install() self.IS_PRO = self._mode == InstallMode.PRO self.IS_NORMAL = self._mode == InstallMode.NORMAL self.PACKAGE_PATH = os.path.join(self.ROOT, self.PACKAGE) self.dependencies = False - self.LINK_FILE = os.path.join(self.MODULE_FILES,'linkpathtomodules.sh') + self.LINK_FILE = os.path.join(self.MODULE_FILES, self.LINK_FILE_NAME) self._create_install_json(self.INSTALL_INFO_PATH) self._validate_installation() self._prepare_infrastructure() @@ -148,122 +145,8 @@ done ''' self.tool.write_file(self.LINK_FILE, linkfile_content) - def get_mpi_info(self): - mpich_info = self.mpich_detector.detect() - if mpich_info:return mpich_info - hmpi_info = self.hmpi_detector.detect() - if hmpi_info:return hmpi_info - openmpi_info = self.openmpi_detector.detect() - if openmpi_info:return openmpi_info - print("MPI not found, please install MPI first.") - sys.exit() - - def check_compiler_mpi(self, compiler_list, compiler_mpi_info): - no_compiler = ["COM","ANY","MISC","APP"] - is_valid = False - compiler_mpi_info = compiler_mpi_info.upper() - valid_list = [] - for compiler in compiler_list: - valid_list.append(compiler) - valid_list.append(f'{compiler}+MPI') - valid_list += no_compiler - for valid_para in valid_list: - if compiler_mpi_info == valid_para: - is_valid = True - break - if not is_valid: - print(f"compiler or mpi info error, Only {valid_list.join('/').lower()} is supported") - sys.exit() - return compiler_mpi_info - - def get_used_compiler(self, compiler_mpi_info): - return compiler_mpi_info.split('+')[0] - - def get_software_type(self,software_name, compiler_mpi_info): - if self.is_mpi_software(software_name): - return SoftwareType.MPI - if compiler_mpi_info == "COM": - return SoftwareType.COMPILER - elif compiler_mpi_info == "ANY": - return SoftwareType.UTIL - elif compiler_mpi_info == "MISC": - return SoftwareType.MISC - elif compiler_mpi_info == "APP": - return SoftwareType.APP - else: - return SoftwareType.LIB - - def get_compiler_info(self, compilers, compiler_mpi_info): - compiler_info = {"cname":None, "cmversion": None, self.FULL_VERSION: None} - for compiler, info_func in compilers.items(): - if compiler in compiler_mpi_info: - compiler_info = info_func() - return compiler_info - - def get_main_version(self, version): - return version.split('.')[0] - - def is_mpi_software(self, software_name): - mpis = ['hmpi', 'openmpi', 'hpcx', 'mpich'] - for mpi in mpis: - if software_name.startswith(mpi): - return True - return False - - def add_mpi_path(self, software_info, install_path): - if not software_info.use_mpi: - return install_path - mpi_info = self.get_mpi_info() - if mpi_info[self.FULL_VERSION] == None: - print("MPI not found!") - return False - mpi_str = mpi_info["name"]+mpi_info[self.FULL_VERSION] - print("Use MPI: "+mpi_str) - if self.IS_PRO: - install_path = os.path.join(install_path, mpi_str) - elif self.IS_NORMAL: - install_path = f"{install_path}-{mpi_str}" - return install_path - - def get_install_path(self, software_info, env_info): - sversion = software_info.full_version - stype = software_info.software_type - cname = env_info['cname'] - cfullver = env_info[self.FULL_VERSION] - sname = software_info.name - if stype == SoftwareType.MPI: - if self.IS_PRO: - mpi_path = os.path.join(self.MPI_PATH, f"{sname}{sversion}-{cname}{cfullver}", sversion) - if self.IS_NORMAL: - mpi_path = os.path.join(self.MPI_PATH,sname, f"{sversion}-{cname}{cfullver}") - print(f"mpi_path:{mpi_path}") - return mpi_path - if stype == SoftwareType.COMPILER: - install_path = os.path.join(self.COMPILER_PATH, f'{sname}/{sversion}') - elif stype == SoftwareType.UTIL: - install_path = os.path.join(self.UTILS_PATH, f'{sname}/{sversion}') - elif stype == SoftwareType.MISC: - install_path = os.path.join(self.MISC_PATH, f'{sname}/{sversion}') - elif stype == SoftwareType.APP: - install_path = os.path.join(self.APP_PATH, f'{sname}/{sversion}-{cname}{cfullver}') - install_path = self.add_mpi_path(software_info, install_path) - else: - if self.IS_PRO: - # install library - install_path = os.path.join(self.LIBS_PATH, cname+cfullver) - # get mpi name and version - install_path = self.add_mpi_path(software_info, install_path) - install_path = os.path.join(install_path, f'{sname}/{sversion}') - elif self.IS_NORMAL: - install_path = os.path.join(self.LIBS_PATH, f'{sname}/{sversion}-{cname}{cfullver}') - install_path = self.add_mpi_path(software_info, install_path) - return install_path - - def is_contained_mpi(self, compiler_mpi_info): - return "MPI" in compiler_mpi_info - def get_files(self, abs_path): - file_list = [d for d in glob(abs_path+'/**', recursive=True)] + file_list = [d for d in glob(str(abs_path)+'/**', recursive=True)] return file_list def add_special_library_path(self, install_path, sname, libs_dir): @@ -311,7 +194,10 @@ done sys.exit(0) return " ".join(depsmap.values()) - def get_module_file_content(self, install_path, sname, sversion): + def get_module_file_content(self, install_path, software_info): + sname = software_info.name + sversion = software_info.full_version + stype = software_info.software_type module_file_content = '' if "hpckit" in sname.lower(): module_file_content = f'''#%Module1.0##################################################################### @@ -354,7 +240,7 @@ prepend-path MODULEPATH {install_path}/HPCKit/latest/modulefiles compiler_values = "setenv CC gcc \nsetenv CXX g++ \nsetenv FC gfortran \nsetenv F77 gfortran \nsetenv F90 gfortran " elif "hmpi" in sname or "openmpi" in sname: compiler_values = "setenv CC mpicc \nsetenv CXX mpicxx \nsetenv FC mpifort \nsetenv F77 mpifort \nsetenv F90 mpifort " - if self.is_mpi_software(sname): + if stype == SoftwareType.MPI: opal_prefix = f"setenv OPAL_PREFIX {install_path}" pmix_install_prefix = f"setenv PMIX_INSTALL_PREFIX {install_path}" depend_content='' @@ -392,9 +278,10 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} sname = software_info.name sversion = software_info.full_version stype = software_info.software_type - cname = env_info['cname'] - cfullversion = env_info[self.FULL_VERSION] - module_file_content = self.get_module_file_content(install_path, sname, sversion) + if stype == SoftwareType.MPI or stype == SoftwareType.APP or stype == SoftwareType.LIB: + cname = env_info.compiler_name + cfullversion = env_info.compiler_version + module_file_content = self.get_module_file_content(install_path, software_info) if not self.is_installed(install_path): return '' # if install_path is empty, The module file should not generated. @@ -446,8 +333,7 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} BASE_PATH = self.MODULE_DEPS_PATH compiler_str = cname + cfullversion if software_info.use_mpi: - mpi_info = self.get_mpi_info() - mpi_str = mpi_info['name'] + mpi_info[self.FULL_VERSION] + mpi_str = env_info.mpi_name + env_info.mpi_version if self.IS_PRO: module_path = os.path.join(self.BASE_PATH, f"{compiler_str}-{mpi_str}" ,sname) if self.IS_NORMAL: @@ -544,37 +430,19 @@ chmod +x {install_script} software_dict['module_path'] = '' self.json.set(install_path, software_dict, True) - def remove_prefix(self, software_path): - if software_path.startswith('package/') or software_path.startswith('./'): - software_path = software_path.replace('./', '', 1) - software_path = software_path.replace('package/', '', 1) - return software_path - def install(self, install_args, isapp = False): software_path = install_args[0] compiler_mpi_info = install_args[1] other_args = install_args[2:] self.tool.prt_content("INSTALL " + software_path) - compilers = {"GCC":self.gcc_detector.detect, "CLANG":self.clang_detector.detect, - "NVC":self.nvcc_detector.detect, "ICC":self.icc_detector.detect, - "BISHENG": self.clang_detector.detect} - compiler_mpi_info = self.check_compiler_mpi(compilers.keys(), compiler_mpi_info) software_info = self.software_factory.create_profile(software_path, compiler_mpi_info, isapp) + env_info = self.env_factory.create_profile(software_info, compiler_mpi_info) stype = software_info.software_type - # get compiler name and version - env_info = self.get_compiler_info(compilers, compiler_mpi_info) - if stype == SoftwareType.LIB or stype == SoftwareType.MPI or stype == SoftwareType.APP: - cmversion = env_info['cmversion'] - cfullver = env_info[self.FULL_VERSION] - if cmversion == None: - print(f"The specified {software_info.compiler_name} Compiler not found!") - return False - else: - print(f"Use Compiler: {env_info['cname']} {cfullver}") - # get install path - install_path = self.get_install_path(software_info, env_info) + install_path = self.path_factory.get_builder(stype).build_path(software_info, env_info) + install_path = str(install_path) + print(install_path) if not install_path: return else: diff --git a/src/installTypes.py b/src/installTypes.py index 3bbf56a..a51a084 100644 --- a/src/installTypes.py +++ b/src/installTypes.py @@ -6,13 +6,5 @@ class InstallMode(Enum): PRO = "0" NORMAL = "1" -class SoftwareCategory(Enum): - COMPILER = 1 - MPI = 2 - UTIL = 3 - LIB = 4 - MISC = 5 - APP = 6 - VersionInfo = Tuple[str, str] # (major_version, full_version) diff --git a/src/jsonService.py b/src/jsonService.py index 4f7e05d..7cb5117 100644 --- a/src/jsonService.py +++ b/src/jsonService.py @@ -67,11 +67,16 @@ class JSONService: :param auto_save: 是否自动保存到文件 :raises TypeError: 当键名不是字符串时抛出 """ + key = self.path_to_str(key) if not isinstance(key, str): raise TypeError("键名必须是字符串类型") self._data[key] = value if auto_save: self.save() + + def path_to_str(self, obj) -> str: + """若对象是 Path 类型则返回字符串，否则返回原对象""" + return str(obj) if isinstance(obj, Path) else obj def delete(self, key: str, auto_save: bool = False) -> bool: """ diff --git a/src/pathBuilder.py b/src/pathBuilder.py index e6b63a8..e42ded6 100644 --- a/src/pathBuilder.py +++ b/src/pathBuilder.py @@ -1,16 +1,30 @@ from abc import ABC, abstractmethod from pathlib import Path -from softwareTypes import SoftwareProfile, EnvironmentProfile +from softwareTypes import SoftwareType, SoftwareProfile, EnvironmentProfile from collections import defaultdict +from deploymentConfig import DeploymentConfig +from collections import defaultdict +import os class PathBuilder(ABC): - def __init__(self, base_config: 'DeploymentConfig'): + def __init__(self, base_config: DeploymentConfig): self.config = base_config @abstractmethod def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: pass + def add_mpi_path(self, software_info, env_info, install_path): + if not software_info.use_mpi: + return install_path + mpi_str = env_info.mpi_name + env_info.mpi_version + print("Use MPI: "+mpi_str) + if self.config.is_pro: + install_path = os.path.join(install_path, mpi_str) + else: + install_path = f"{install_path}-{mpi_str}" + return install_path + class MpiPathBuilder(PathBuilder): def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: folder_pattern = ( @@ -21,33 +35,49 @@ class MpiPathBuilder(PathBuilder): def _construct_mpi_path(self, pattern: str, software: SoftwareProfile, env: EnvironmentProfile) -> Path: path_vars = { - 'name': software.name, - 'version': software.full_version, - 'compiler': env.compiler_name, - 'compiler_ver': env.compiler_version - } + 'name': software.name, + 'version': software.full_version, + 'compiler': env.compiler_name, + 'compiler_ver': env.compiler_version + } return self.config.mpi_base / pattern.format(**path_vars) class CompilerPathBuilder(PathBuilder): def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: - return self.config.compiler_base / f"{software.name}/{software.version}" + return self.config.compiler_base / f"{software.name}/{software.full_version}" + +class UtilPathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + util_path = self.config.util_base / f"{software.name}/{software.full_version}" + return util_path.absolute() + +class MiscPathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + return self.config.misc_base / f"{software.name}/{software.full_version}" class LibPathBuilder(PathBuilder): def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: - base_path = self.config.pro_lib_base if self.config.is_pro else self.config.normal_lib_base - path_segments = [ - f"{env.compiler_name}{env.compiler_version}", - *([f"mpi{env.mpi_version}"] if software.requires_mpi else []), - f"{software.name}/{software.version}" - ] - return base_path.joinpath(*path_segments) + base_path = self.config.lib_base + if self.config.is_pro: + lib_path = os.path.join(base_path, f"{env.compiler_name}{env.compiler_version}") + lib_path = self.add_mpi_path(software, env, lib_path) + lib_path = os.path.join(lib_path, f"{software.name}/{software.full_version}") + else: + lib_path = os.path.join(base_path, f"{software.name}/{software.full_version}-{env.compiler_name}{env.compiler_version}") + lib_path = self.add_mpi_path(software, env, lib_path) + return lib_path -from collections import defaultdict +class AppPathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + base_path = self.config.app_base + app_path = os.path.join(base_path, f"{software.name}/{software.full_version}-{env.compiler_name}{env.compiler_version}") + app_path = self.add_mpi_path(software, env, app_path) + return app_path class PathFactory: _builder_registry = defaultdict(lambda: None) - def __init__(self, config: 'DeploymentConfig'): + def __init__(self, config: DeploymentConfig): self.config = config self._initialize_builders() @@ -62,8 +92,8 @@ class PathFactory: }) def get_builder(self, software_type: SoftwareType) -> PathBuilder: - - if builder := self._builder_registry[software_type]: + builder = self._builder_registry[software_type] + if builder: return builder raise ValueError(f"No path builder registered for {software_type}") diff --git a/src/softwareFactory.py b/src/softwareFactory.py index 166a801..3b8973b 100644 --- a/src/softwareFactory.py +++ b/src/softwareFactory.py @@ -14,18 +14,19 @@ class SoftwareFactory: 'default': DefaultStrategy() } - def create_profile(self, software_path: str, compiler_info: str, isapp=False) -> SoftwareProfile: + def create_profile(self, software_path: str, compiler_mpi_info: str, isapp=False) -> SoftwareProfile: """创建软件信息对象的统一入口""" + compiler_mpi_info = compiler_mpi_info.upper() (software_name, version, suffix) = self._parse_name_and_version(software_path) - software_type = self._determine_type(software_name, compiler_info, isapp) + software_type = self._determine_type(software_name, compiler_mpi_info, isapp) profile = SoftwareProfile( name=self._handle_suffix(software_name, suffix), full_version=version, major_version=self._parse_major_version(version), software_type=software_type, suffix=suffix, - use_mpi='+MPI' in compiler_info, - compiler_name=compiler_info.split('+')[0] if '+' in compiler_info else compiler_info, + use_mpi='+MPI' in compiler_mpi_info, + compiler_name=compiler_mpi_info.split('+')[0] if '+' in compiler_mpi_info else compiler_mpi_info, install_script_path=self._get_install_script_path(software_name, version, software_type, suffix), ) return profile diff --git a/src/softwareTypes.py b/src/softwareTypes.py index bfee193..120424e 100644 --- a/src/softwareTypes.py +++ b/src/softwareTypes.py @@ -25,3 +25,6 @@ class EnvironmentProfile: compiler_name: str compiler_version: str compiler_major_version: str + mpi_name: str = None + mpi_version: str = None + mpi_major_version: str = None -- Gitee From 93a8de938d2abd7c9c0209e7089222bbd315f8e0 Mon Sep 17 00:00:00 2001 From: chunlin Date: Mon, 28 Apr 2025 15:28:16 +0800 Subject: [PATCH 2/6] reconstruct modulefile auto generate --- data.config | 6 +- src/deploymentConfig.py | 27 ++++++--- src/installService.py | 126 +++++++++++++--------------------------- src/pathBuilder.py | 66 ++++++++++++++++++++- 4 files changed, 125 insertions(+), 100 deletions(-) diff --git a/data.config b/data.config index 7f98b04..37cd683 100644 --- a/data.config +++ b/data.config @@ -36,9 +36,9 @@ case_dir = ${JARVIS_JOBSCRIPT}/QE/ set -e set -x #$PATCH_TOOL $JARVIS_TPL/top50/qe/6.4.1/opt.patch -./configure F90=flang F77=flang MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp --prefix=$1 -make -j pw -make install +#./configure F90=flang F77=flang MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp --prefix=$1 +#make -j pw +#make install [CLEAN] make clean diff --git a/src/deploymentConfig.py b/src/deploymentConfig.py index bf0c279..61a5fb6 100644 --- a/src/deploymentConfig.py +++ b/src/deploymentConfig.py @@ -10,16 +10,25 @@ class DeploymentConfig: self.lib_base = Path(paths['LIBS_PATH']) self.util_base = Path(paths['UTILS_PATH']) self.app_base = Path(paths['APP_PATH']) - self.misc_base = Path(paths['MISC_PATH']) + self.misc_base = Path(paths['MISC_PATH']) + + #基础module路径配置 + if not self.is_pro: + self.mpi_module_base = Path(paths['MODULE_MPI_PATH']) + self.compiler_module_base = Path(paths['MODULE_COMPILER_PATH']) + self.lib_module_base = Path(paths['MODULE_LIB_PATH']) + self.util_module_base = Path(paths['MODULE_UTIL_PATH']) + self.app_module_base = Path(paths['MODULE_APP_PATH']) + self.misc_module_base = Path(paths['MODULE_MISC_PATH']) + else: + self.mpi_module_base = Path(paths['MODULE_DEPS_PATH']) + self.compiler_module_base = Path(paths['MODULE_FILES']) + self.lib_module_base = Path(paths['MODULE_DEPS_PATH']) + self.util_module_base = Path(paths['MODULE_FILES']) + self.app_module_base = Path(paths['MODULE_APP_PATH']) + self.misc_module_base = Path(paths['MODULE_FILES']) # 路径模式配置 self.pro_path_pattern = "{name}{version}-{compiler}{compiler_ver}/{version}" self.normal_path_pattern = "{name}/{version}-{compiler}{compiler_ver}" - @property - def pro_mpi_path_pattern(self) -> Path: - return self.pro_path_pattern - - @property - def normal_mpi_path_pattern(self) -> Path: - return self.normal_path_pattern - + self.pro_module_path_pattern = "{compiler}{compiler_ver}/{name}/{version}" diff --git a/src/installService.py b/src/installService.py index a695db9..25e907b 100644 --- a/src/installService.py +++ b/src/installService.py @@ -194,10 +194,13 @@ done sys.exit(0) return " ".join(depsmap.values()) - def get_module_file_content(self, install_path, software_info): + def get_module_file_content(self, install_path, software_info, env_info): sname = software_info.name sversion = software_info.full_version stype = software_info.software_type + if stype == SoftwareType.MPI or stype == SoftwareType.APP or stype == SoftwareType.LIB: + cname = env_info.compiler_name + cfullversion = env_info.compiler_version module_file_content = '' if "hpckit" in sname.lower(): module_file_content = f'''#%Module1.0##################################################################### @@ -265,6 +268,20 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} {libs_str} {incs_str} ''' + if self.IS_PRO: + if stype == SoftwareType.MPI: + compiler_str = cname + cfullversion + software_str = sname + sversion + attach_module_path = os.path.join(self.MODULE_DEPS_PATH, software_str) + self.tool.mkdirs(attach_module_path) + module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" + print(f'attach module file {attach_module_path} successfully generated.') + elif stype == SoftwareType.COMPILER: + module_path = os.path.join(self.MODULE_FILES, sname) + attach_module_path = os.path.join(self.MODULE_DEPS_PATH, software_str) + self.tool.mkdirs(attach_module_path) + module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" + print(f'attach module file {attach_module_path} successfully generated.') return module_file_content def is_installed(self, install_path): @@ -274,89 +291,6 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} return True return self.json.get(install_path) - def gen_module_file(self, install_path, software_info, env_info): - sname = software_info.name - sversion = software_info.full_version - stype = software_info.software_type - if stype == SoftwareType.MPI or stype == SoftwareType.APP or stype == SoftwareType.LIB: - cname = env_info.compiler_name - cfullversion = env_info.compiler_version - module_file_content = self.get_module_file_content(install_path, software_info) - if not self.is_installed(install_path): - return '' - # if install_path is empty, The module file should not generated. - if len(os.listdir(install_path)) == 0: - print('module file did not generated because no file generated under install path') - return '' - if stype == SoftwareType.MPI: - compiler_str = cname + cfullversion - software_str = sname + sversion - if self.IS_PRO: - module_path = os.path.join(self.MODULE_DEPS_PATH, compiler_str ,sname) - attach_module_path = os.path.join(self.MODULE_DEPS_PATH, compiler_str+'-'+software_str) - self.tool.mkdirs(attach_module_path) - module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" - print(f'attach module file {attach_module_path} successfully generated.') - elif self.IS_NORMAL: - module_path = os.path.join(self.MODULE_COMPILER_PATH, sname, f'{sversion}-{compiler_str}') - else: - if stype == SoftwareType.COMPILER: - software_str = sname + sversion - if self.IS_PRO: - module_path = os.path.join(self.MODULE_FILES, sname) - attach_module_path = os.path.join(self.MODULE_DEPS_PATH, software_str) - self.tool.mkdirs(attach_module_path) - module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" - print(f'attach module file {attach_module_path} successfully generated.') - elif self.IS_NORMAL: - module_path = os.path.join(self.MODULE_COMPILER_PATH, sname, sversion) - elif stype == SoftwareType.UTIL: - if self.IS_PRO: - module_path = os.path.join(self.MODULE_FILES, sname) - elif self.IS_NORMAL: - module_path = os.path.join(self.MODULE_UTIL_PATH, sname, sversion) - elif stype == SoftwareType.MISC: - if self.IS_PRO: - module_path = os.path.join(self.MODULE_FILES, sname) - elif self.IS_NORMAL: - module_path = os.path.join(self.MODULE_MISC_PATH, sname, sversion) - else: - if self.IS_NORMAL: - if stype == SoftwareType.APP: - BASE_PATH = self.MODULE_APP_PATH - else: - BASE_PATH = self.MODULE_LIB_PATH - elif self.IS_PRO: - if stype == SoftwareType.APP: - BASE_PATH = self.MODULE_APP_PATH - else: - BASE_PATH = self.MODULE_DEPS_PATH - compiler_str = cname + cfullversion - if software_info.use_mpi: - mpi_str = env_info.mpi_name + env_info.mpi_version - if self.IS_PRO: - module_path = os.path.join(self.BASE_PATH, f"{compiler_str}-{mpi_str}" ,sname) - if self.IS_NORMAL: - module_path = os.path.join(BASE_PATH, sname, f"{sversion}-{compiler_str}-{mpi_str}") - else: - if self.IS_PRO: - module_path = os.path.join(self.BASE_PATH, compiler_str, sname) - elif self.IS_NORMAL: - module_path = os.path.join(BASE_PATH, sname, f"{sversion}-{compiler_str}") - if self.IS_PRO: - self.tool.mkdirs(module_path) - module_file = os.path.join(module_path, sversion) - elif self.IS_NORMAL: - self.tool.mkdirs(os.path.dirname(module_path)) - module_file = module_path - self.tool.write_file(module_file, module_file_content) - print(f"module file {module_file} successfully generated") - row = self.json.get(install_path) - row["module_path"] = module_file - self.json.set(install_path, row, True) - #更新linkfile - if self.IS_NORMAL:self.update_modules() - def update_modules(self): upd_cmd = f''' cd {self.MODULE_FILES} @@ -429,6 +363,25 @@ chmod +x {install_script} software_dict['version'] = software_info.full_version software_dict['module_path'] = '' self.json.set(install_path, software_dict, True) + + def _write_modulefile(self, install_path, module_path, content): + base_name = os.path.basename(module_path) + self.tool.mkdirs(base_name) + self.tool.write_file(module_path, content) + print(f"module file {module_path} successfully generated") + row = self.json.get(install_path) + row["module_path"] = module_path + self.json.set(install_path, row, True) + #更新linkfile + if self.IS_NORMAL:self.update_modules() + + def gen_module_file(self, install_path, software_info, env_info): + stype = software_info.software_type + module_file_content = self.get_module_file_content(install_path, software_info, env_info) + # gen module file + module_path = self.path_factory.get_module_builder(stype).build_path(software_info, env_info) + module_path = str(module_path) + self._write_modulefile(install_path, module_path, module_file_content) def install(self, install_args, isapp = False): software_path = install_args[0] @@ -457,8 +410,9 @@ chmod +x {install_script} self.install_package(software_info.install_script_path, install_path, other_args) # add install info self.add_install_info(software_info, install_path) - # gen module file - self.gen_module_file( install_path, software_info, env_info) + if not self.is_installed(install_path): + return '' + self.gen_module_file(install_path, software_info, env_info) def install_depend(self): depend_file = self.ds.get_depend_file() diff --git a/src/pathBuilder.py b/src/pathBuilder.py index e42ded6..5e82a27 100644 --- a/src/pathBuilder.py +++ b/src/pathBuilder.py @@ -74,9 +74,58 @@ class AppPathBuilder(PathBuilder): app_path = self.add_mpi_path(software, env, app_path) return app_path +class MpiModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + folder_pattern = ( + self.config.pro_module_path_pattern if self.config.is_pro + else self.config.normal_path_pattern + ) + return self._construct_mpi_path(folder_pattern, software, env) + + def _construct_mpi_path(self, pattern: str, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + path_vars = { + 'name': software.name, + 'version': software.full_version, + 'compiler': env.compiler_name, + 'compiler_ver': env.compiler_version + } + return self.config.mpi_module_base / pattern.format(**path_vars) + +class CompilerModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + return self.config.compiler_module_base / f"{software.name}/{software.full_version}" + +class UtilModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + return self.config.util_module_base / f"{software.name}/{software.full_version}" + +class MiscModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + return self.config.misc_module_base / f"{software.name}/{software.full_version}" + +class LibModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + base_path = self.config.lib_module_base + if self.config.is_pro: + if software.use_mpi: + lib_path = os.path.join(base_path, f"{env.compiler_name}{env.compiler_version}-{env.mpi_name}{env.mpi_version}", software.name, software.full_version) + else: + lib_path = os.path.join(base_path, f"{env.compiler_name}{env.compiler_version}", software.name, software.full_version) + else: + lib_path = os.path.join(base_path, f"{software.name}/{software.full_version}-{env.compiler_name}{env.compiler_version}") + lib_path = self.add_mpi_path(software, env, lib_path) + return lib_path + +class AppModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + base_path = self.config.app_module_base + app_path = os.path.join(base_path, f"{software.name}/{software.full_version}-{env.compiler_name}{env.compiler_version}") + app_path = self.add_mpi_path(software, env, app_path) + return app_path + class PathFactory: _builder_registry = defaultdict(lambda: None) - + _builder_module_registry = defaultdict(lambda: None) def __init__(self, config: DeploymentConfig): self.config = config self._initialize_builders() @@ -90,10 +139,23 @@ class PathFactory: SoftwareType.UTIL: UtilPathBuilder(self.config), SoftwareType.MISC: MiscPathBuilder(self.config) }) - + self._builder_module_registry.update({ + SoftwareType.MPI: MpiModulePathBuilder(self.config), + SoftwareType.COMPILER: CompilerModulePathBuilder(self.config), + SoftwareType.LIB: LibModulePathBuilder(self.config), + SoftwareType.APP: AppModulePathBuilder(self.config), + SoftwareType.UTIL: UtilModulePathBuilder(self.config), + SoftwareType.MISC: MiscModulePathBuilder(self.config) + }) def get_builder(self, software_type: SoftwareType) -> PathBuilder: builder = self._builder_registry[software_type] if builder: return builder raise ValueError(f"No path builder registered for {software_type}") + def get_module_builder(self, software_type: SoftwareType) -> PathBuilder: + builder = self._builder_module_registry[software_type] + if builder: + return builder + raise ValueError(f"No path builder registered for {software_type}") + -- Gitee From 6d14b85113bb0f97d935aa13e1e59319c90f510b Mon Sep 17 00:00:00 2001 From: chunlin Date: Mon, 28 Apr 2025 20:47:38 +0800 Subject: [PATCH 3/6] recontruct modulefile --- src/dependencyManager.py | 16 +++++ src/envVarGenerator.py | 17 +++++ src/generationStrategy.py | 107 +++++++++++++++++++++++++++++ src/installService.py | 141 +++++++++----------------------------- src/moduleConfig.py | 15 ++++ src/moduleController.py | 47 +++++++++++++ src/pathManager.py | 19 +++++ 7 files changed, 253 insertions(+), 109 deletions(-) create mode 100644 src/dependencyManager.py create mode 100644 src/envVarGenerator.py create mode 100644 src/generationStrategy.py create mode 100644 src/moduleConfig.py create mode 100644 src/moduleController.py create mode 100644 src/pathManager.py diff --git a/src/dependencyManager.py b/src/dependencyManager.py new file mode 100644 index 0000000..8a4eea8 --- /dev/null +++ b/src/dependencyManager.py @@ -0,0 +1,16 @@ +class DependencyResolver: + def __init__(self, required_deps): + self.required = required_deps + + def generate_load_commands(self) -> str: + if not self.required: + return "" + required_deps = " ".join(self.required) + depend_content=''' +foreach dep {%s} { + if { ![is-loaded $dep] } { + module load $dep + } +} +''' % required_deps + return depend_content diff --git a/src/envVarGenerator.py b/src/envVarGenerator.py new file mode 100644 index 0000000..273669c --- /dev/null +++ b/src/envVarGenerator.py @@ -0,0 +1,17 @@ +class EnvVarGenerator: + COMPILER_MAP = { + "gcc": {"CC": "gcc", "CXX": "g++"}, + "bisheng": {"CC": "clang", "FC": "flang"} + } + + @classmethod + def get_compiler_vars(cls, compiler_name: str) -> str: + vars = cls.COMPILER_MAP.get(compiler_name, {}) + return "\n".join(f"setenv {k} {v}" for k, v in vars.items()) + + @staticmethod + def generate_mpi_vars() -> str: + return """setenv MPICC mpicc +setenv MPICXX mpicxx +setenv MPIF90 mpifort""" + diff --git a/src/generationStrategy.py b/src/generationStrategy.py new file mode 100644 index 0000000..f64a099 --- /dev/null +++ b/src/generationStrategy.py @@ -0,0 +1,107 @@ +from abc import ABC, abstractmethod +from moduleConfig import ModuleConfig + +class IGenerationStrategy(ABC): + @abstractmethod + def generate_header(self, config: ModuleConfig) -> str: + pass + + @abstractmethod + def generate_body(self, config: ModuleConfig) -> str: + pass + + @abstractmethod + def generate_footer(self, config: ModuleConfig) -> str: + pass + +class BaseStrategy(IGenerationStrategy): + def generate_header(self, config: ModuleConfig) -> str: + return f"""#%Module1.0 +set prefix {config.install_root} +set version {config.full_version} +""" + + def generate_body(self, config: ModuleConfig) -> str: + sname = config.software_name.upper().replace('-', '_') + return f""" +setenv {sname}_PATH $prefix +""" + + def generate_footer(self, config: ModuleConfig) -> str: + return "" + +class CompilerStrategy(BaseStrategy): + def generate_body(self, config: ModuleConfig) -> str: + return f""" +prepend-path MANPATH $prefix/share/man +""" + + def generate_footer(self, config: ModuleConfig) -> str: + if config.is_pro: + return f""" +prepend-path MODULEPATH {config.attach_module_path} +""" + + +class GCCStrategy(CompilerStrategy): + def generate_body(self, config: ModuleConfig) -> str: + base = super().generate_body(config) + return base + """ +setenv CC gcc +setenv CXX g++ +setenv FC gfortran +""" + +class ClangStrategy(CompilerStrategy): + def generate_body(self, config: ModuleConfig) -> str: + base = super().generate_body(config) + return base + """ +setenv CC clang +setenv CXX clang++ +setenv FC flang +""" + +class MPIStrategy(BaseStrategy): + def generate_body(self, config: ModuleConfig) -> str: + return f""" +setenv MPI_HOME $prefix + +setenv OMPI_CC $env(CC) +setenv OMPI_CXX $env(CXX) +setenv OPAL_PREFIX {config.install_root} +setenv PMIX_INSTALL_PREFIX {config.install_root} +setenv MPICC mpicc +setenv MPICXX mpicxx +setenv MPIF90 mpifort +""" + + def generate_footer(self, config: ModuleConfig) -> str: + if config.is_pro: + return f""" +prepend-path MODULEPATH {config.attach_module_path} +""" + + +class HPCKitStrategy(BaseStrategy): + def generate_footer(self, config: ModuleConfig) -> str: + return f""" +prepend-path MODULEPATH {config.install_root}/HPCKit/latest/modulefiles +""" + +class KMLStrategy(BaseStrategy): + def generate_body(self, config: ModuleConfig) -> str: + base = super().generate_body(config) + libs_dir = [ + "$prefix/lib/kblas/nolocking", + "$prefix/lib/kblas/pthread", + "$prefix/lib/kblas/omp", + "$prefix/lib/kblas/locking", + "$prefix/lib/kvml/single", + "$prefix/lib/kvml/multi", + "$prefix/lib/kspblas/single", + "$prefix/lib/kspblas/multi" + ] + return base + f""" +prepend-path LD_LIBRARY_PATH {':'.join(libs_dir)} +""" + diff --git a/src/installService.py b/src/installService.py index 25e907b..7c42914 100644 --- a/src/installService.py +++ b/src/installService.py @@ -7,6 +7,7 @@ import fnmatch from enum import Enum from glob import glob from typing import Tuple, Optional, Dict +from pathlib import Path from dataService import DataService from toolService import ToolService @@ -20,6 +21,8 @@ from envFactory import EnvFactory from softwareTypes import SoftwareType from pathBuilder import PathFactory from deploymentConfig import DeploymentConfig +from moduleConfig import ModuleConfig +from moduleController import ModulefileEngine class Singleton(type): @@ -145,23 +148,6 @@ done ''' self.tool.write_file(self.LINK_FILE, linkfile_content) - def get_files(self, abs_path): - file_list = [d for d in glob(str(abs_path)+'/**', recursive=True)] - return file_list - - def add_special_library_path(self, install_path, sname, libs_dir): - prefix = install_path.replace(install_path, "$prefix") - if "kml" in sname: - libs_dir.append(os.path.join(prefix, "lib/kblas/nolocking")) - libs_dir.append(os.path.join(prefix, "lib/kblas/pthread")) - libs_dir.append(os.path.join(prefix, "lib/kblas/omp")) - libs_dir.append(os.path.join(prefix, "lib/kblas/locking")) - libs_dir.append(os.path.join(prefix, "lib/kvml/single")) - libs_dir.append(os.path.join(prefix, "lib/kvml/multi")) - libs_dir.append(os.path.join(prefix, "lib/kspblas/single")) - libs_dir.append(os.path.join(prefix, "lib/kspblas/multi")) - return libs_dir - def get_loaded_modules(self): import subprocess try: @@ -194,96 +180,6 @@ done sys.exit(0) return " ".join(depsmap.values()) - def get_module_file_content(self, install_path, software_info, env_info): - sname = software_info.name - sversion = software_info.full_version - stype = software_info.software_type - if stype == SoftwareType.MPI or stype == SoftwareType.APP or stype == SoftwareType.LIB: - cname = env_info.compiler_name - cfullversion = env_info.compiler_version - module_file_content = '' - if "hpckit" in sname.lower(): - module_file_content = f'''#%Module1.0##################################################################### -prepend-path MODULEPATH {install_path}/HPCKit/latest/modulefiles -''' - return module_file_content - file_list = self.get_files(install_path) - bins_dir_type = ["bin"] - libs_dir_type = ["libs", "lib", "lib64"] - incs_dir_type = ["include"] - bins_dir = [] - libs_dir = [] - incs_dir = [] - compiler_values = '' - bins_str = '' - libs_str = '' - incs_str = '' - opal_prefix = '' - pmix_install_prefix = '' - for file in file_list: - if not os.path.isdir(file): - continue - last_dir = file.split('/')[-1] - if last_dir in bins_dir_type: - bins_dir.append(file.replace(install_path, "$prefix")) - elif last_dir in libs_dir_type: - libs_dir.append(file.replace(install_path, "$prefix")) - elif last_dir in incs_dir_type: - incs_dir.append(file.replace(install_path, "$prefix")) - self.add_special_library_path(install_path, sname, libs_dir) - if len(bins_dir) >= 1: - bins_str = "prepend-path PATH "+':'.join(bins_dir) - if len(libs_dir) >= 1: - libs_str = "prepend-path LD_LIBRARY_PATH "+':'.join(libs_dir) - if len(incs_dir) >= 1: - incs_str = "prepend-path INCLUDE " + ':'.join(incs_dir) - if "bisheng" in sname: - compiler_values = "setenv CC clang \nsetenv CXX clang++ \nsetenv FC flang \nsetenv F77 flang \nsetenv F90 flang " - elif "gcc" in sname: - compiler_values = "setenv CC gcc \nsetenv CXX g++ \nsetenv FC gfortran \nsetenv F77 gfortran \nsetenv F90 gfortran " - elif "hmpi" in sname or "openmpi" in sname: - compiler_values = "setenv CC mpicc \nsetenv CXX mpicxx \nsetenv FC mpifort \nsetenv F77 mpifort \nsetenv F90 mpifort " - if stype == SoftwareType.MPI: - opal_prefix = f"setenv OPAL_PREFIX {install_path}" - pmix_install_prefix = f"setenv PMIX_INSTALL_PREFIX {install_path}" - depend_content='' - if self.dependencies: - # complement dependencies - depend_content=''' -foreach dep {%s} { - if { ![is-loaded $dep] } { - module load $dep - } -} -''' % self.dependencies - module_file_content = f'''#%Module1.0##################################################################### -set prefix {install_path} -set version {sversion} -{depend_content} -setenv {sname.upper().replace('-','_')}_PATH {install_path} -{compiler_values} -{opal_prefix} -{pmix_install_prefix} -{bins_str} -{libs_str} -{incs_str} -''' - if self.IS_PRO: - if stype == SoftwareType.MPI: - compiler_str = cname + cfullversion - software_str = sname + sversion - attach_module_path = os.path.join(self.MODULE_DEPS_PATH, software_str) - self.tool.mkdirs(attach_module_path) - module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" - print(f'attach module file {attach_module_path} successfully generated.') - elif stype == SoftwareType.COMPILER: - module_path = os.path.join(self.MODULE_FILES, sname) - attach_module_path = os.path.join(self.MODULE_DEPS_PATH, software_str) - self.tool.mkdirs(attach_module_path) - module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" - print(f'attach module file {attach_module_path} successfully generated.') - return module_file_content - def is_installed(self, install_path): #为了兼容老版本，只要安装路径下存在installed也算做已安装 installed_file_path = os.path.join(install_path, "installed") @@ -374,11 +270,38 @@ chmod +x {install_script} self.json.set(install_path, row, True) #更新linkfile if self.IS_NORMAL:self.update_modules() + + def _get_attach_module_path(self, software_info, env_info): + if self.IS_NORMAL: + return "" + stype = software_info.software_type + attach_module_path = "" + if stype == SoftwareType.MPI: + attach_module_path = os.path.join(self.MODULE_DEPS_PATH, f"{env_info.compiler_name}{env_info.compiler_version}-{software_info.name}{software_info.full_version}") + self.tool.mkdirs(attach_module_path) + elif stype == SoftwareType.COMPILER: + attach_module_path = os.path.join(self.MODULE_DEPS_PATH, f"{software_info.name}{software_info.full_version}") + self.tool.mkdirs(attach_module_path) + return attach_module_path def gen_module_file(self, install_path, software_info, env_info): stype = software_info.software_type - module_file_content = self.get_module_file_content(install_path, software_info, env_info) - # gen module file + # 构建配置对象 + deps = self.dependencies.split() if self.dependencies else [] + attach_mod_path = self._get_attach_module_path(software_info, env_info) + config = ModuleConfig( + is_pro = self.IS_PRO, + install_root=Path(install_path), + software_name=software_info.name, + full_version=software_info.full_version, + category=stype, + dependencies=deps, + attach_module_path=attach_mod_path + ) + + # 生成模块文件 + engine = ModulefileEngine() + module_file_content = engine.generate(config) module_path = self.path_factory.get_module_builder(stype).build_path(software_info, env_info) module_path = str(module_path) self._write_modulefile(install_path, module_path, module_file_content) diff --git a/src/moduleConfig.py b/src/moduleConfig.py new file mode 100644 index 0000000..61beded --- /dev/null +++ b/src/moduleConfig.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from pathlib import Path +from softwareTypes import SoftwareType +from typing import Tuple, Optional, Dict, List + +@dataclass +class ModuleConfig: + is_pro:bool = True + install_root: Path = None + software_name: str = "" + full_version: str = "" + category: SoftwareType = None + dependencies: List[str] = None + attach_module_path: str = "" + diff --git a/src/moduleController.py b/src/moduleController.py new file mode 100644 index 0000000..1cf7eb2 --- /dev/null +++ b/src/moduleController.py @@ -0,0 +1,47 @@ +from pathlib import Path +from generationStrategy import IGenerationStrategy, BaseStrategy, GCCStrategy, ClangStrategy, MPIStrategy, HPCKitStrategy, KMLStrategy +from pathManager import PathOrganizer +from envVarGenerator import EnvVarGenerator +from softwareTypes import SoftwareType +from moduleConfig import ModuleConfig +from dependencyManager import DependencyResolver + +class ModulefileEngine: + STRATEGY_MAP = { + "gcc": GCCStrategy(), + "kgcc": GCCStrategy(), + "clang": ClangStrategy(), + "bisheng": ClangStrategy(), + "openmpi": MPIStrategy(), + "hmpi": MPIStrategy(), + "hpckit": HPCKitStrategy(), + "kml": KMLStrategy() + } + + def __init__(self, strategy: IGenerationStrategy = None): + self.strategy = strategy or BaseStrategy() + + def generate(self, config: ModuleConfig) -> str: + self.strategy = self.STRATEGY_MAP.get(config.software_name, BaseStrategy()) + sections = [ + self.strategy.generate_header(config), + self.strategy.generate_body(config), + self._generate_dependencies(config), + self._generate_paths(config), + self.strategy.generate_footer(config) + ] + return "\n".join(filter(None, sections)) + + def _generate_paths(self, config: ModuleConfig) -> str: + if config.software_name == "hpckit": + return "" + path_config = PathOrganizer.auto_discover(config.install_root) + return f""" +prepend-path PATH {PathOrganizer.format_paths(path_config['bins'])} +prepend-path LD_LIBRARY_PATH {PathOrganizer.format_paths(path_config['libs'])} +prepend-path C_INCLUDE_PATH {PathOrganizer.format_paths(path_config['includes'])} +""" + + def _generate_dependencies(self, config: ModuleConfig) -> str: + return DependencyResolver(config.dependencies).generate_load_commands() + diff --git a/src/pathManager.py b/src/pathManager.py new file mode 100644 index 0000000..e5485b6 --- /dev/null +++ b/src/pathManager.py @@ -0,0 +1,19 @@ +from pathlib import Path + +class PathOrganizer: + install_path = None + @staticmethod + def auto_discover(install_path: Path): + PathOrganizer.install_path = install_path + return { + "bins": sorted([p for p in install_path.glob("**/bin") if p.is_dir()]), + "libs": sorted([p for p in install_path.glob("**/lib*") if p.is_dir()]), + "includes": sorted([p for p in install_path.glob("**/include*") if p.is_dir()]) + } + + @staticmethod + def format_paths(paths) -> str: + if PathOrganizer.install_path is None: + raise ValueError("install_path is not set. Please call auto_discover first.") + return ":".join(f"$prefix/{p.relative_to(PathOrganizer.install_path)}" for p in paths) + -- Gitee From f55051dda262cdc9b16a0708bc7bd789f8df6cb7 Mon Sep 17 00:00:00 2001 From: chunlin Date: Tue, 29 Apr 2025 11:35:17 +0800 Subject: [PATCH 4/6] remove useless files --- data.config | 6 +++--- src/buildService.py | 11 ++++++----- src/deployService.py | 33 --------------------------------- src/envFactory.py | 1 + src/envVarGenerator.py | 17 ----------------- src/generationStrategy.py | 6 ++++-- src/installService.py | 20 ++++++++------------ src/installTypes.py | 9 ++++++++- src/moduleController.py | 5 +++-- 9 files changed, 33 insertions(+), 75 deletions(-) delete mode 100644 src/deployService.py delete mode 100644 src/envVarGenerator.py diff --git a/data.config b/data.config index 37cd683..7f98b04 100644 --- a/data.config +++ b/data.config @@ -36,9 +36,9 @@ case_dir = ${JARVIS_JOBSCRIPT}/QE/ set -e set -x #$PATCH_TOOL $JARVIS_TPL/top50/qe/6.4.1/opt.patch -#./configure F90=flang F77=flang MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp --prefix=$1 -#make -j pw -#make install +./configure F90=flang F77=flang MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp --prefix=$1 +make -j pw +make install [CLEAN] make clean diff --git a/src/buildService.py b/src/buildService.py index b9b9ec6..0c5372e 100644 --- a/src/buildService.py +++ b/src/buildService.py @@ -4,6 +4,7 @@ from dataService import DataService from executeService import ExecuteService from toolService import ToolService from commandBuilder import CommandBuilder +from installTypes import InstallProfile from installService import InstallService import subprocess import os @@ -51,13 +52,13 @@ class BuildService: app_version = self.ds.get_app_version() or self.DEFAULT_APP_VERSION app_compiler = self.ds.get_app_compiler() or self.DEFAULT_COMPILER - result = self.installService.install([f"{app_name}/{app_version}", app_compiler], True) - self._handle_install_result(result) + install_profile = self.installService.install([f"{app_name}/{app_version}", app_compiler], True) + self._handle_install_result(install_profile) def _handle_install_result(self, result): - install_path = result["install_path"] - software_info = result["software_info"] - env_info = result["env_info"] + install_path = result.install_path + software_info = result.software_info + env_info = result.env_info build_cmd = self.command.build() self._execute_script(self.get_build_file(), build_cmd, "build", install_path) self.installService.add_install_info(software_info, install_path) diff --git a/src/deployService.py b/src/deployService.py deleted file mode 100644 index d8b6f35..0000000 --- a/src/deployService.py +++ /dev/null @@ -1,33 +0,0 @@ -from pathBuilder import PathFactory -from deploymentConfig import DeploymentConfig -from softwareTypes import SoftwareType, SoftwareProfile, EnvironmentProfile -from installTypes import InstallMode - -class DeploymentService: - def __init__(self, config: DeploymentConfig): - self.factory = PathFactory(config) - self.config = config - - def resolve_install_path( - self, - software: SoftwareProfile, - env: EnvironmentProfile - ) -> Path: - """生成规范化安装路径""" - self._validate_input(software, env) - - builder = self.factory.get_builder(software.software_type) - path = builder.build_path(software, env) - return path.absolute() - - def _validate_input(self, software: SoftwareProfile, env: EnvironmentProfile): - if software.software_type == SoftwareType.MPI and not env.mpi_version: - raise ValueError("MPI software requires MPI version in environment profile") - - if software.requires_mpi and not env.mpi_version: - raise ValueError("MPI-dependent software requires MPI version specification") - -if __name__ == '__main__': - dconfig = DeploymentConfig(InstallMode.NORMAL) - dservice = DeploymentService() - diff --git a/src/envFactory.py b/src/envFactory.py index ad1b045..de315b6 100644 --- a/src/envFactory.py +++ b/src/envFactory.py @@ -1,4 +1,5 @@ import os +import sys from typing import Dict from softwareTypes import SoftwareProfile, SoftwareType, EnvironmentProfile from detectorService import GCCDetector, ClangDetector,NVCCDetector,ICCDetector diff --git a/src/envVarGenerator.py b/src/envVarGenerator.py deleted file mode 100644 index 273669c..0000000 --- a/src/envVarGenerator.py +++ /dev/null @@ -1,17 +0,0 @@ -class EnvVarGenerator: - COMPILER_MAP = { - "gcc": {"CC": "gcc", "CXX": "g++"}, - "bisheng": {"CC": "clang", "FC": "flang"} - } - - @classmethod - def get_compiler_vars(cls, compiler_name: str) -> str: - vars = cls.COMPILER_MAP.get(compiler_name, {}) - return "\n".join(f"setenv {k} {v}" for k, v in vars.items()) - - @staticmethod - def generate_mpi_vars() -> str: - return """setenv MPICC mpicc -setenv MPICXX mpicxx -setenv MPIF90 mpifort""" - diff --git a/src/generationStrategy.py b/src/generationStrategy.py index f64a099..1907701 100644 --- a/src/generationStrategy.py +++ b/src/generationStrategy.py @@ -32,7 +32,8 @@ setenv {sname}_PATH $prefix class CompilerStrategy(BaseStrategy): def generate_body(self, config: ModuleConfig) -> str: - return f""" + base = super().generate_body(config) + return base + f""" prepend-path MANPATH $prefix/share/man """ @@ -63,7 +64,8 @@ setenv FC flang class MPIStrategy(BaseStrategy): def generate_body(self, config: ModuleConfig) -> str: - return f""" + base = super().generate_body(config) + return base + f""" setenv MPI_HOME $prefix setenv OMPI_CC $env(CC) diff --git a/src/installService.py b/src/installService.py index 7c42914..98ef32c 100644 --- a/src/installService.py +++ b/src/installService.py @@ -14,7 +14,7 @@ from toolService import ToolService from executeService import ExecuteService from jsonService import JSONService from installStrategy import PathStrategyFactory -from installTypes import InstallMode +from installTypes import InstallMode, InstallProfile from versionParser import VersionParser from softwareFactory import SoftwareFactory from envFactory import EnvFactory @@ -318,23 +318,19 @@ chmod +x {install_script} # get install path install_path = self.path_factory.get_builder(stype).build_path(software_info, env_info) install_path = str(install_path) - print(install_path) if not install_path: return - else: - self.tool.mkdirs(install_path) - if isapp: - return { - "install_path": install_path, - "software_info":software_info, - "env_info":env_info - } + self.tool.mkdirs(install_path) + install_profile = InstallProfile( + install_path = install_path, + software_info = software_info, + env_info = env_info + ) + if isapp: return install_profile # get install script self.install_package(software_info.install_script_path, install_path, other_args) # add install info self.add_install_info(software_info, install_path) - if not self.is_installed(install_path): - return '' self.gen_module_file(install_path, software_info, env_info) def install_depend(self): diff --git a/src/installTypes.py b/src/installTypes.py index a51a084..9eb80c0 100644 --- a/src/installTypes.py +++ b/src/installTypes.py @@ -1,10 +1,17 @@ -# types.py +from dataclasses import dataclass from enum import Enum from typing import Tuple, Optional, Dict +from softwareTypes import SoftwareProfile, EnvironmentProfile class InstallMode(Enum): PRO = "0" NORMAL = "1" +@dataclass +class InstallProfile: + install_path: str = "" + software_info: SoftwareProfile = None + env_info: EnvironmentProfile = None + VersionInfo = Tuple[str, str] # (major_version, full_version) diff --git a/src/moduleController.py b/src/moduleController.py index 1cf7eb2..3bb83ea 100644 --- a/src/moduleController.py +++ b/src/moduleController.py @@ -1,7 +1,6 @@ from pathlib import Path from generationStrategy import IGenerationStrategy, BaseStrategy, GCCStrategy, ClangStrategy, MPIStrategy, HPCKitStrategy, KMLStrategy from pathManager import PathOrganizer -from envVarGenerator import EnvVarGenerator from softwareTypes import SoftwareType from moduleConfig import ModuleConfig from dependencyManager import DependencyResolver @@ -15,7 +14,8 @@ class ModulefileEngine: "openmpi": MPIStrategy(), "hmpi": MPIStrategy(), "hpckit": HPCKitStrategy(), - "kml": KMLStrategy() + "kml-gcc": KMLStrategy(), + "kml-bisheng": KMLStrategy() } def __init__(self, strategy: IGenerationStrategy = None): @@ -40,6 +40,7 @@ class ModulefileEngine: prepend-path PATH {PathOrganizer.format_paths(path_config['bins'])} prepend-path LD_LIBRARY_PATH {PathOrganizer.format_paths(path_config['libs'])} prepend-path C_INCLUDE_PATH {PathOrganizer.format_paths(path_config['includes'])} +prepend-path INCLUDE {PathOrganizer.format_paths(path_config['includes'])} """ def _generate_dependencies(self, config: ModuleConfig) -> str: -- Gitee From 56a11136ebc4edb493e2a14b34600f08b3e6e750 Mon Sep 17 00:00:00 2001 From: chunlin Date: Tue, 29 Apr 2025 17:02:00 +0800 Subject: [PATCH 5/6] fix module file bugs --- data.config | 9 +-- dev/README.txt | 1 + init.sh | 1 + src/commandBuilder.py | 7 ++- src/dataService.py | 3 + src/installService.py | 4 +- src/moduleController.py | 28 ++++++++- templates/top50/gromacs.2023.3.config | 91 +++++++++++++++++++++++++++ templates/top50/qe.6.4.1.config | 78 +++++++++++++++++++++++ test-blast.sh => test/test-blast.sh | 0 10 files changed, 212 insertions(+), 10 deletions(-) create mode 100644 dev/README.txt create mode 100644 templates/top50/gromacs.2023.3.config create mode 100644 templates/top50/qe.6.4.1.config rename test-blast.sh => test/test-blast.sh (100%) diff --git a/data.config b/data.config index 7f98b04..fb3aa31 100644 --- a/data.config +++ b/data.config @@ -13,6 +13,7 @@ module use $JARVIS_MODULES/modules module load hpckit/2025.3.30 module load bisheng/compiler4.2.0/bishengmodule module load bisheng/hmpi25.0.0/hmpi +cd $JARVIS_DEV [ ! -d q-e-qe-6.4.1 ] && tar -xzvf ${JARVIS_DOWNLOAD}/q-e-qe-6.4.1.tar.gz exit 0 @@ -22,15 +23,15 @@ module use $JARVIS_MODULES/modules module load hpckit/2025.3.30 module load bisheng/compiler4.2.0/bishengmodule module load bisheng/hmpi25.0.0/hmpi -export QE_ROOT=${JARVIS_ROOT}/q-e-qe-6.4.1/ +export QE_ROOT=${JARVIS_DEV}/q-e-qe-6.4.1/ [APP] app_name = QE app_version = 6.4.1 compiler = bisheng+mpi -build_dir = ${JARVIS_ROOT}/q-e-qe-6.4.1/ -binary_dir = ${JARVIS_ROOT}/q-e-qe-6.4.1/bin/ -case_dir = ${JARVIS_JOBSCRIPT}/QE/ +build_dir = ${JARVIS_DEV}/q-e-qe-6.4.1/ +binary_dir = ${JARVIS_DEV}/q-e-qe-6.4.1/bin/ +case_dir = ${JARVIS_JOBSCRIPT}/QE/6.4.1/ [BUILD] set -e diff --git a/dev/README.txt b/dev/README.txt new file mode 100644 index 0000000..420e611 --- /dev/null +++ b/dev/README.txt @@ -0,0 +1 @@ +HPC apps development dir diff --git a/init.sh b/init.sh index 862035a..6c9f5d9 100644 --- a/init.sh +++ b/init.sh @@ -41,6 +41,7 @@ elif [ "$JARVIS_MODE" -eq 1 ]; then export JARVIS_JOBSCRIPT=${JARVIS_SOFT_ROOT}/jobscript fi export JARVIS_TMP=/tmp +export JARVIS_DEV=${CUR_PATH}/dev export JARVIS_DOWNLOAD=${CUR_PATH}/downloads export JARVIS_TMP_DOWNLOAD=${CUR_PATH}/tmp export JARVIS_EXE=${CUR_PATH}/exe diff --git a/src/commandBuilder.py b/src/commandBuilder.py index b9ebf75..68c6e6d 100644 --- a/src/commandBuilder.py +++ b/src/commandBuilder.py @@ -49,9 +49,9 @@ class CommandBuilder: # 拼接二进制路径和参数 binary_path = os.path.join( self.ds.app_config.binary_dir, - self.ds.binary_file[0] + self.ds.binary_file ) - return f'{cmd} {binary_path} {self.ds.binary_file[1]}' + return f'{cmd} {binary_path} {self.ds.get_binary_para()}' def full_run(self) -> str: """完整运行环境命令链""" @@ -77,10 +77,11 @@ class CommandBuilder: def job_run(self, num): job_file_path = self.ds.get_job_run_file() - print(f"start job run {self.ds.get_app_name}") + print(f"start job run {self.ds.get_app_name()}") job_cmd = self.ds.get_job_cmd() if num == 1 else self.ds.get_job2_cmd() job_content = f''' {self.env_activation()} +mkdir -p {self.ds.app_config.case_dir} cd {self.ds.app_config.case_dir} cat > job_run.sh << \EOF {job_cmd} diff --git a/src/dataService.py b/src/dataService.py index 50128e2..08ca9ea 100644 --- a/src/dataService.py +++ b/src/dataService.py @@ -283,3 +283,6 @@ class DataService(metaclass=Singleton): def get_binary_file(self): return self.binary_file + + def get_binary_para(self): + return self.binary_para diff --git a/src/installService.py b/src/installService.py index 98ef32c..5ba955e 100644 --- a/src/installService.py +++ b/src/installService.py @@ -261,8 +261,8 @@ chmod +x {install_script} self.json.set(install_path, software_dict, True) def _write_modulefile(self, install_path, module_path, content): - base_name = os.path.basename(module_path) - self.tool.mkdirs(base_name) + base_path = os.path.dirname(module_path) + self.tool.mkdirs(base_path) self.tool.write_file(module_path, content) print(f"module file {module_path} successfully generated") row = self.json.get(install_path) diff --git a/src/moduleController.py b/src/moduleController.py index 3bb83ea..a4610b4 100644 --- a/src/moduleController.py +++ b/src/moduleController.py @@ -32,7 +32,7 @@ class ModulefileEngine: ] return "\n".join(filter(None, sections)) - def _generate_paths(self, config: ModuleConfig) -> str: + def _generate_paths2(self, config: ModuleConfig) -> str: if config.software_name == "hpckit": return "" path_config = PathOrganizer.auto_discover(config.install_root) @@ -43,6 +43,32 @@ prepend-path C_INCLUDE_PATH {PathOrganizer.format_paths(path_config['includes']) prepend-path INCLUDE {PathOrganizer.format_paths(path_config['includes'])} """ + def _generate_paths(self, config: ModuleConfig) -> str: + if config.software_name == "hpckit": + return "" + + path_config = PathOrganizer.auto_discover(config.install_root) + path_commands = [] + + # 环境变量与路径配置的映射关系 + env_mapping = { + 'PATH': ('bins', lambda x: x), + 'LD_LIBRARY_PATH': ('libs', lambda x: x), + 'C_INCLUDE_PATH': ('includes', lambda x: x), + 'INCLUDE': ('includes', lambda x: x) + } + + # 动态生成有效路径指令 + for env_var, (config_key, formatter) in env_mapping.items(): + paths = path_config.get(config_key, []) + if not paths: + continue + + formatted_paths = PathOrganizer.format_paths(formatter(paths)) + path_commands.append(f"prepend-path {env_var} {formatted_paths}") + + return "\n".join(path_commands) if path_commands else "" + def _generate_dependencies(self, config: ModuleConfig) -> str: return DependencyResolver(config.dependencies).generate_load_commands() diff --git a/templates/top50/gromacs.2023.3.config b/templates/top50/gromacs.2023.3.config new file mode 100644 index 0000000..0c25abc --- /dev/null +++ b/templates/top50/gromacs.2023.3.config @@ -0,0 +1,91 @@ +[SERVER] +11.11.11.11 + +[DOWNLOAD] +gromacs/2023.3 http://ftp.gromacs.org/pub/gromacs/gromacs-2023.3.tar.gz +Testcase https://repository.prace-ri.eu/ueabs/GROMACS/2.2/GROMACS_TestCaseC.tar.xz + +[DEPENDENCY] +set -e +set -x +module purge +module use $JARVIS_MODULES/modules +./jarvis -install cmake/3.28.2 any +./jarvis -install hpckit/2025.3.30 any +module load hpckit/2025.3.30 +module load bisheng/compiler4.2.0/bishengmodule +./jarvis -install openblas/0.3.14 any +module load bisheng/hmpi25.0.0/hmpi +export CC=mpicc CXX=mpicxx FC=mpifort +./jarvis -install fftw/3.3.10 clang+mpi +#解压源码 +cd $JARVIS_DEV +if [ ! -d gromacs-2023.3 ]; then + tar -xvf ${JARVIS_DOWNLOAD}/gromacs-2023.3.tar.gz +fi +#解压算例 +CASE_DIR=${JARVIS_JOBSCRIPT}/gromacs/2023.3 +if [ ! -d $CASE_DIR ]; then + mkdir -p $CASE_DIR + cd $CASE_DIR + tar -xf ${JARVIS_DOWNLOAD}/GROMACS_TestCaseC.tar.xz +fi + +[ENV] +module purge +module use $JARVIS_MODULES/modules +module load cmake/3.28.2 +module load hpckit/2025.3.30 +#低版本GCC会报错 +module add gcc/compiler12.3.1/gccmodule +module load bisheng/compiler4.2.0/bishengmodule +module load bisheng/hmpi25.0.0/hmpi +module load openblas/0.3.14 +module load fftw/3.3.10-bisheng4.2.0-hmpi25.0.0 +compile_dir=$(dirname $(dirname $(which clang))) +export gcc_compile_dir=$(dirname $(dirname $(which gcc))) +export gromacs_dir=$JARVIS_DEV/gromacs-2023.3 + +[APP] +app_name = gromacs +app_version = 2023.3 +compiler = bisheng+mpi +build_dir = $JARVIS_DEV/gromacs-2023.3 +binary_dir = $JARVIS_DEV/gromacs-2023.3/build/bin +case_dir = ${JARVIS_JOBSCRIPT}/gromacs/2023.3/GROMACS_TestCaseC + +[BUILD] +cmake --version +mkdir -p build +cd build +FLAGS="-mcpu=linxicore9100 -O3 -ffast-math -mllvm --aarch64-sched-inline-asm=false -mllvm -unroll-threshold-aggressive=600" +LD_FLAGS="-mcpu=linxicore9100 -O3 -L${gcc_compile_dir}/lib64 -lgfortran -lbsmath -lflang -L${compile_dir}/lib/jemalloc-64kbpage -ljemalloc" + +CFLAGS=$FLAGS CXXFLAGS=$FLAGS LDFLAGS=$LD_FLAGS CC=mpicc CXX=mpicxx FC=mpifort cmake -DCMAKE_INSTALL_PREFIX=$1 -DBUILD_SHARED_LIBS=on -DBUILD_TESTING=on -DREGRESSIONTEST_DOWNLOAD=off -DGMX_BUILD_OWN_FFTW=off -DGMX_SIMD=ARM_SVE -DGMX_SIMD_ARM_SVE_LENGTH=256 -DGMX_DOUBLE=off -DGMX_EXTERNAL_BLAS=on -DGMX_EXTERNAL_LAPACK=on -DGMX_FFT_LIBRARY=fftw3 -DGMX_BLAS_USER=${OPENBLAS_PATH}/lib/libopenblas.a -DGMX_LAPACK_USER=${OPENBLAS_PATH}/lib/libopenblas.a -DFFTWF_LIBRARY=$FFTW_PATH/lib/libfftw3f.so -DFFTWF_INCLUDE_DIR=$FFTW_PATH/include -DGMX_X11=off -DGMX_OPENMP=off -DGMX_MPI=on -DGMX_THREAD_MPI=off -DGMX_CYCLE_SUBCOUNTERS=off ../ + +make -j40 V=1 +make install + +[CLEAN] +rm -rf build + +[RUN] +run = mpirun -np 574 --allow-run-as-root -x UCX_TLS=sm --bind-to cpulist:ordered -mca pml ucx -mca btl ^vader,tcp,openib,uct +binary = gmx_mpi mdrun -dlb yes -v -nsteps 4000 -noconfout -pin on -pinoffset 0 -ntomp 1 -npme 112 -g md_sve_0229-ucpg-bisheng-2P.log -s stmv.28M.tpr +nodes = 1 + +[JOB] +#!/bin/bash +#DSUB -n gmx +#DSUB --job_type cosched +#DSUB -N 1 +#DSUB -R "cpu=128" +#DSUB -o gmx_%J.log +#DSUB -e gmx_err_%J.log +#DSUB -T '2h' + +export HOSTFILE=hostfile.gmx +rm -f $HOSTFILE +touch $HOSTFILE +cat ${CCSCHEDULER_ALLOC_FILE} | sort > $HOSTFILE +mpirun -np 128 --hostfile $HOSTFILE -x UCX_TLS=sm --bind-to cpulist:ordered -mca pml ucx -mca btl ^vader,tcp,openib,uct ${gromacs_dir}/build/bin/gmx_mpi mdrun -dlb yes -v -nsteps 4000 -noconfout -pin on -pinoffset 0 -ntomp 1 -npme 8 -g 3-5-pme8-128C.log -s stmv.28M.tpr diff --git a/templates/top50/qe.6.4.1.config b/templates/top50/qe.6.4.1.config new file mode 100644 index 0000000..fb3aa31 --- /dev/null +++ b/templates/top50/qe.6.4.1.config @@ -0,0 +1,78 @@ +[SERVER] +11.11.11.11 + +[DOWNLOAD] +qe/6.4 $JARVIS_PROXY/QEF/q-e/archive/refs/tags/qe-6.4.1.tar.gz q-e-qe-6.4.1.tar.gz + +[DEPENDENCY] +set -e +set -x +module purge +module use $JARVIS_MODULES/modules +./jarvis -install hpckit/2025.3.30 any +module load hpckit/2025.3.30 +module load bisheng/compiler4.2.0/bishengmodule +module load bisheng/hmpi25.0.0/hmpi +cd $JARVIS_DEV +[ ! -d q-e-qe-6.4.1 ] && tar -xzvf ${JARVIS_DOWNLOAD}/q-e-qe-6.4.1.tar.gz +exit 0 + +[ENV] +module purge +module use $JARVIS_MODULES/modules +module load hpckit/2025.3.30 +module load bisheng/compiler4.2.0/bishengmodule +module load bisheng/hmpi25.0.0/hmpi +export QE_ROOT=${JARVIS_DEV}/q-e-qe-6.4.1/ + +[APP] +app_name = QE +app_version = 6.4.1 +compiler = bisheng+mpi +build_dir = ${JARVIS_DEV}/q-e-qe-6.4.1/ +binary_dir = ${JARVIS_DEV}/q-e-qe-6.4.1/bin/ +case_dir = ${JARVIS_JOBSCRIPT}/QE/6.4.1/ + +[BUILD] +set -e +set -x +#$PATCH_TOOL $JARVIS_TPL/top50/qe/6.4.1/opt.patch +./configure F90=flang F77=flang MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp --prefix=$1 +make -j pw +make install + +[CLEAN] +make clean + +[RUN] +run = mpirun --allow-run-as-root -x OMP_NUM_THREADS=1 -np 8 +binary = pw.x -input test_3.in +nodes = 1 + +[BATCH] +mpirun --allow-run-as-root -x OMP_NUM_THREADS=1 -np 8 $QE_ROOT/bin/pw.x -input test_3.in + +[JOB] +#!/bin/sh +#DSUB -n qe_test +#DSUB --job_type cosched:hmpi +#DSUB -A root.default +#DSUB -q root.default +#DSUB -N 1 +#DSUB -R cpu=128 +#DSUB -oo qe.%J.out +#DSUB -eo qe.%J.err + +##set runtime environment variables +module load QE +ulimit -s unlimited +ulimit -c unlimited +echo "----HOSTFILE generated---" +cat $CCS_HOST_FILE +echo "-------------------------" +EXEC_CMD="time -p mpirun $CCS_MPI_OPTIONS -n 32 -x OMP_NUM_THREADS=1 -mca pml ucx -mca btl ^vader,tcp,openib,uct -x UCX_TLS=self,sm,rc pw.x -input test_3.in" +echo "$EXEC_CMD" +date +$EXEC_CMD +date + diff --git a/test-blast.sh b/test/test-blast.sh similarity index 100% rename from test-blast.sh rename to test/test-blast.sh -- Gitee From 72783e47f7d3689c973442e4b3a5ffbe18577415 Mon Sep 17 00:00:00 2001 From: chunlin Date: Tue, 29 Apr 2025 17:29:18 +0800 Subject: [PATCH 6/6] add templates& no env without run&build --- .gitignore | 3 ++- src/buildService.py | 4 +++- src/envService.py | 1 - src/runService.py | 5 +++++ templates/top50/gromacs.2023.3.config | 4 +--- templates/top50/qe.6.4.1.config | 3 +-- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index c6f0b2d..153f023 100644 --- a/.gitignore +++ b/.gitignore @@ -71,4 +71,5 @@ installed app/* test.sh *.swp -workloads/* \ No newline at end of file +workloads/* +dev diff --git a/src/buildService.py b/src/buildService.py index 0c5372e..870cbc8 100644 --- a/src/buildService.py +++ b/src/buildService.py @@ -6,6 +6,7 @@ from toolService import ToolService from commandBuilder import CommandBuilder from installTypes import InstallProfile from installService import InstallService +from envService import EnvService import subprocess import os @@ -20,6 +21,7 @@ class BuildService: self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() + self.env_service = EnvService() self.command = CommandBuilder() self.installService = InstallService() @@ -37,12 +39,12 @@ class BuildService: clean_root_path = self.ds.root_path return os.path.join(clean_root_path, self.CLEAN_FILE_NAME) - def clean(self): clean_cmd = self.ds.get_clean_cmd() self._execute_script(self.get_clean_file(), clean_cmd, "clean") def inject_env(self): + self.env_service.env() env_cmd = self.command.env_activation() self.exe.exec_inject(env_cmd) diff --git a/src/envService.py b/src/envService.py index 0bcc5a2..7ce764c 100644 --- a/src/envService.py +++ b/src/envService.py @@ -21,7 +21,6 @@ class EnvService(object,metaclass=Singleton): self.command = CommandBuilder() # 注入命令生成组件 self.exe = ExecuteService() self.ds = DataService() - self.env() def env(self): print(f"set environment...{self.ds.get_app_name()}") diff --git a/src/runService.py b/src/runService.py index 5e3f634..7b86802 100644 --- a/src/runService.py +++ b/src/runService.py @@ -5,11 +5,13 @@ import sys from toolService import ToolService from executeService import ExecuteService from dataService import DataService +from envService import EnvService from commandBuilder import CommandBuilder class RunService: def __init__(self): self.ds = DataService() + self.env_service = EnvService() self.exe = ExecuteService() self.tool = ToolService() self.command = CommandBuilder() # 注入命令生成组件 @@ -32,13 +34,16 @@ class RunService: print(f"start run {self.ds.get_app_name()}") nodes = int(self.ds.get_run_cmd('nodes')) self.gen_hostfile(nodes) + self.env_service.env() run_cmd = self.command.full_run() self.exe.exec_raw(run_cmd) def batch_run(self): + self.env_service.env() batch_cmd = self.command.batch_run() self.exe.exec_raw(run_cmd) def job_run(self,num): + self.env_service.env() job_cmd = self.command.job_run(num) self.exe.exec_raw(job_cmd) diff --git a/templates/top50/gromacs.2023.3.config b/templates/top50/gromacs.2023.3.config index 0c25abc..40abd5f 100644 --- a/templates/top50/gromacs.2023.3.config +++ b/templates/top50/gromacs.2023.3.config @@ -20,9 +20,7 @@ export CC=mpicc CXX=mpicxx FC=mpifort ./jarvis -install fftw/3.3.10 clang+mpi #解压源码 cd $JARVIS_DEV -if [ ! -d gromacs-2023.3 ]; then - tar -xvf ${JARVIS_DOWNLOAD}/gromacs-2023.3.tar.gz -fi +tar -xvf ${JARVIS_DOWNLOAD}/gromacs-2023.3.tar.gz #解压算例 CASE_DIR=${JARVIS_JOBSCRIPT}/gromacs/2023.3 if [ ! -d $CASE_DIR ]; then diff --git a/templates/top50/qe.6.4.1.config b/templates/top50/qe.6.4.1.config index fb3aa31..a2a79cc 100644 --- a/templates/top50/qe.6.4.1.config +++ b/templates/top50/qe.6.4.1.config @@ -14,8 +14,7 @@ module load hpckit/2025.3.30 module load bisheng/compiler4.2.0/bishengmodule module load bisheng/hmpi25.0.0/hmpi cd $JARVIS_DEV -[ ! -d q-e-qe-6.4.1 ] && tar -xzvf ${JARVIS_DOWNLOAD}/q-e-qe-6.4.1.tar.gz -exit 0 +tar -xzvf ${JARVIS_DOWNLOAD}/q-e-qe-6.4.1.tar.gz [ENV] module purge -- Gitee