diff --git a/.gitignore b/.gitignore index c6f0b2d87af21aa24d620699e95a344b84cd14b5..153f0235419cc48b8fb87588ea72c89c960996d5 100644 --- a/.gitignore +++ b/.gitignore @@ -71,4 +71,5 @@ installed app/* test.sh *.swp -workloads/* \ No newline at end of file +workloads/* +dev diff --git a/data.config b/data.config index 7f98b048d3991279a435ccf565be4f13c3955d7b..fb3aa31478861cc88e9d874581e994d7c3f3916b 100644 --- a/data.config +++ b/data.config @@ -13,6 +13,7 @@ module use $JARVIS_MODULES/modules module load hpckit/2025.3.30 module load bisheng/compiler4.2.0/bishengmodule module load bisheng/hmpi25.0.0/hmpi +cd $JARVIS_DEV [ ! -d q-e-qe-6.4.1 ] && tar -xzvf ${JARVIS_DOWNLOAD}/q-e-qe-6.4.1.tar.gz exit 0 @@ -22,15 +23,15 @@ module use $JARVIS_MODULES/modules module load hpckit/2025.3.30 module load bisheng/compiler4.2.0/bishengmodule module load bisheng/hmpi25.0.0/hmpi -export QE_ROOT=${JARVIS_ROOT}/q-e-qe-6.4.1/ +export QE_ROOT=${JARVIS_DEV}/q-e-qe-6.4.1/ [APP] app_name = QE app_version = 6.4.1 compiler = bisheng+mpi -build_dir = ${JARVIS_ROOT}/q-e-qe-6.4.1/ -binary_dir = ${JARVIS_ROOT}/q-e-qe-6.4.1/bin/ -case_dir = ${JARVIS_JOBSCRIPT}/QE/ +build_dir = ${JARVIS_DEV}/q-e-qe-6.4.1/ +binary_dir = ${JARVIS_DEV}/q-e-qe-6.4.1/bin/ +case_dir = ${JARVIS_JOBSCRIPT}/QE/6.4.1/ [BUILD] set -e diff --git a/dev/README.txt b/dev/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..420e6112b1701f8b2127b38224101960c274688a --- /dev/null +++ b/dev/README.txt @@ -0,0 +1 @@ +HPC apps development dir diff --git a/init.sh b/init.sh index 862035a86c584a3a43ac814b7f560ae86851bf97..6c9f5d9e8e9b59608828af8f84528f31d8882a48 100644 --- a/init.sh +++ b/init.sh @@ -41,6 +41,7 @@ elif [ "$JARVIS_MODE" -eq 1 ]; then export JARVIS_JOBSCRIPT=${JARVIS_SOFT_ROOT}/jobscript fi export JARVIS_TMP=/tmp +export JARVIS_DEV=${CUR_PATH}/dev export JARVIS_DOWNLOAD=${CUR_PATH}/downloads export JARVIS_TMP_DOWNLOAD=${CUR_PATH}/tmp export JARVIS_EXE=${CUR_PATH}/exe diff --git a/src/buildService.py b/src/buildService.py index b9b9ec62e21ec6f7571d28f7d257e0852bdbade9..870cbc83d21a2ae70251afd3bea026a304646835 100644 --- a/src/buildService.py +++ b/src/buildService.py @@ -4,7 +4,9 @@ from dataService import DataService from executeService import ExecuteService from toolService import ToolService from commandBuilder import CommandBuilder +from installTypes import InstallProfile from installService import InstallService +from envService import EnvService import subprocess import os @@ -19,6 +21,7 @@ class BuildService: self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() + self.env_service = EnvService() self.command = CommandBuilder() self.installService = InstallService() @@ -36,12 +39,12 @@ class BuildService: clean_root_path = self.ds.root_path return os.path.join(clean_root_path, self.CLEAN_FILE_NAME) - def clean(self): clean_cmd = self.ds.get_clean_cmd() self._execute_script(self.get_clean_file(), clean_cmd, "clean") def inject_env(self): + self.env_service.env() env_cmd = self.command.env_activation() self.exe.exec_inject(env_cmd) @@ -51,13 +54,13 @@ class BuildService: app_version = self.ds.get_app_version() or self.DEFAULT_APP_VERSION app_compiler = self.ds.get_app_compiler() or self.DEFAULT_COMPILER - result = self.installService.install([f"{app_name}/{app_version}", app_compiler], True) - self._handle_install_result(result) + install_profile = self.installService.install([f"{app_name}/{app_version}", app_compiler], True) + self._handle_install_result(install_profile) def _handle_install_result(self, result): - install_path = result["install_path"] - software_info = result["software_info"] - env_info = result["env_info"] + install_path = result.install_path + software_info = result.software_info + env_info = result.env_info build_cmd = self.command.build() self._execute_script(self.get_build_file(), build_cmd, "build", install_path) self.installService.add_install_info(software_info, install_path) diff --git a/src/commandBuilder.py b/src/commandBuilder.py index b9ebf75a195214db6265b1e52046bd93b9d4b95d..68c6e6d678248c5df44145e653bbfeec01095b92 100644 --- a/src/commandBuilder.py +++ b/src/commandBuilder.py @@ -49,9 +49,9 @@ class CommandBuilder: # 拼接二进制路径和参数 binary_path = os.path.join( self.ds.app_config.binary_dir, - self.ds.binary_file[0] + self.ds.binary_file ) - return f'{cmd} {binary_path} {self.ds.binary_file[1]}' + return f'{cmd} {binary_path} {self.ds.get_binary_para()}' def full_run(self) -> str: """完整运行环境命令链""" @@ -77,10 +77,11 @@ class CommandBuilder: def job_run(self, num): job_file_path = self.ds.get_job_run_file() - print(f"start job run {self.ds.get_app_name}") + print(f"start job run {self.ds.get_app_name()}") job_cmd = self.ds.get_job_cmd() if num == 1 else self.ds.get_job2_cmd() job_content = f''' {self.env_activation()} +mkdir -p {self.ds.app_config.case_dir} cd {self.ds.app_config.case_dir} cat > job_run.sh << \EOF {job_cmd} diff --git a/src/dataService.py b/src/dataService.py index 50128e24781ca6b245bd5f78c0fd467c569450d0..08ca9ead05c4e79559343a5467df078a036b10b2 100644 --- a/src/dataService.py +++ b/src/dataService.py @@ -283,3 +283,6 @@ class DataService(metaclass=Singleton): def get_binary_file(self): return self.binary_file + + def get_binary_para(self): + return self.binary_para diff --git a/src/dependencyManager.py b/src/dependencyManager.py new file mode 100644 index 0000000000000000000000000000000000000000..8a4eea87e4553732ba54fc10d0ebc8466ec77e53 --- /dev/null +++ b/src/dependencyManager.py @@ -0,0 +1,16 @@ +class DependencyResolver: + def __init__(self, required_deps): + self.required = required_deps + + def generate_load_commands(self) -> str: + if not self.required: + return "" + required_deps = " ".join(self.required) + depend_content=''' +foreach dep {%s} { + if { ![is-loaded $dep] } { + module load $dep + } +} +''' % required_deps + return depend_content diff --git a/src/deploymentConfig.py b/src/deploymentConfig.py new file mode 100644 index 0000000000000000000000000000000000000000..61a5fb6d08ee13267bb9fc5b6418221b65231630 --- /dev/null +++ b/src/deploymentConfig.py @@ -0,0 +1,34 @@ +from pathlib import Path +from installTypes import InstallMode + +class DeploymentConfig: + def __init__(self, mode, paths): + self.is_pro = mode == InstallMode.PRO + # 基础路径配置 + self.mpi_base = Path(paths['MPI_PATH']) + self.compiler_base = Path(paths['COMPILER_PATH']) + self.lib_base = Path(paths['LIBS_PATH']) + self.util_base = Path(paths['UTILS_PATH']) + self.app_base = Path(paths['APP_PATH']) + self.misc_base = Path(paths['MISC_PATH']) + + #基础module路径配置 + if not self.is_pro: + self.mpi_module_base = Path(paths['MODULE_MPI_PATH']) + self.compiler_module_base = Path(paths['MODULE_COMPILER_PATH']) + self.lib_module_base = Path(paths['MODULE_LIB_PATH']) + self.util_module_base = Path(paths['MODULE_UTIL_PATH']) + self.app_module_base = Path(paths['MODULE_APP_PATH']) + self.misc_module_base = Path(paths['MODULE_MISC_PATH']) + else: + self.mpi_module_base = Path(paths['MODULE_DEPS_PATH']) + self.compiler_module_base = Path(paths['MODULE_FILES']) + self.lib_module_base = Path(paths['MODULE_DEPS_PATH']) + self.util_module_base = Path(paths['MODULE_FILES']) + self.app_module_base = Path(paths['MODULE_APP_PATH']) + self.misc_module_base = Path(paths['MODULE_FILES']) + # 路径模式配置 + self.pro_path_pattern = "{name}{version}-{compiler}{compiler_ver}/{version}" + self.normal_path_pattern = "{name}/{version}-{compiler}{compiler_ver}" + + self.pro_module_path_pattern = "{compiler}{compiler_ver}/{name}/{version}" diff --git a/src/envFactory.py b/src/envFactory.py new file mode 100644 index 0000000000000000000000000000000000000000..de315b6afcb70d0a35635c23902e1fa766feeefe --- /dev/null +++ b/src/envFactory.py @@ -0,0 +1,87 @@ +import os +import sys +from typing import Dict +from softwareTypes import SoftwareProfile, SoftwareType, EnvironmentProfile +from detectorService import GCCDetector, ClangDetector,NVCCDetector,ICCDetector +from detectorService import HMPIDetector, OpenMPIDetector, MPICHDetector +from versionParser import VersionParser + +class EnvFactory: + def __init__(self): + self.ROOT = os.getcwd() + self.gcc_detector = GCCDetector() + self.clang_detector = ClangDetector() + self.icc_detector = ICCDetector() + self.nvcc_detector = NVCCDetector() + + self.hmpi_detector = HMPIDetector() + self.openmpi_detector = OpenMPIDetector() + self.mpich_detector = MPICHDetector() + self._compilers = { + "GCC":self.gcc_detector.detect, + "CLANG":self.clang_detector.detect, + "NVC":self.nvcc_detector.detect, + "ICC":self.icc_detector.detect, + "BISHENG": self.clang_detector.detect + } + + def create_profile(self, software: SoftwareProfile, compiler_mpi_info: str) -> EnvironmentProfile: + """创建环境信息对象的统一入口""" + compiler_mpi_info = self._check_compiler_mpi(self._compilers.keys(), compiler_mpi_info) + stype = software.software_type + if stype != SoftwareType.LIB and stype != SoftwareType.MPI and stype != SoftwareType.APP: + return None + compiler_info = self._get_compiler_info(compiler_mpi_info) + profile = EnvironmentProfile( + compiler_name=compiler_info['cname'], + compiler_version=compiler_info['full_version'], + compiler_major_version=compiler_info['cmversion'] + ) + if software.use_mpi: + mpi_info = self._get_mpi_info() + profile.mpi_name = mpi_info['name'] + profile.mpi_version = mpi_info['full_version'] + profile.mpi_major_version = mpi_info['mversion'] + return profile + + def _check_compiler_mpi(self, compiler_list, compiler_mpi_info): + no_compiler = ["COM","ANY","MISC","APP"] + is_valid = False + compiler_mpi_info = compiler_mpi_info.upper() + valid_list = [] + for compiler in compiler_list: + valid_list.append(compiler) + valid_list.append(f'{compiler}+MPI') + valid_list += no_compiler + for valid_para in valid_list: + if compiler_mpi_info == valid_para: + is_valid = True + break + if not is_valid: + print(f"compiler or mpi info error, Only {valid_list.join('/').lower()} is supported") + sys.exit() + return compiler_mpi_info + + def _get_compiler_info(self, compiler_mpi_info): + compiler_info = None + for compiler, info_func in self._compilers.items(): + if compiler in compiler_mpi_info: + compiler_info = info_func() + break + if compiler_info == None: + print(f"The specified {compiler_mpi_info} Compiler not found!") + sys.exit() + cname = compiler_info['cname'] + cfullver = compiler_info['full_version'] + print(f"Use Compiler: {cname} {cfullver}") + return compiler_info + + def _get_mpi_info(self): + mpich_info = self.mpich_detector.detect() + if mpich_info:return mpich_info + hmpi_info = self.hmpi_detector.detect() + if hmpi_info:return hmpi_info + openmpi_info = self.openmpi_detector.detect() + if openmpi_info:return openmpi_info + print("MPI not found, please install MPI first.") + sys.exit() diff --git a/src/envService.py b/src/envService.py index 0bcc5a2158a2291ea6512c3bced6675b8ca90d04..7ce764cf973b19be0973a257f71b9b51dad9ab9f 100644 --- a/src/envService.py +++ b/src/envService.py @@ -21,7 +21,6 @@ class EnvService(object,metaclass=Singleton): self.command = CommandBuilder() # 注入命令生成组件 self.exe = ExecuteService() self.ds = DataService() - self.env() def env(self): print(f"set environment...{self.ds.get_app_name()}") diff --git a/src/generationStrategy.py b/src/generationStrategy.py new file mode 100644 index 0000000000000000000000000000000000000000..190770155a8e7a7342a17eb6527ec2026340a8e7 --- /dev/null +++ b/src/generationStrategy.py @@ -0,0 +1,109 @@ +from abc import ABC, abstractmethod +from moduleConfig import ModuleConfig + +class IGenerationStrategy(ABC): + @abstractmethod + def generate_header(self, config: ModuleConfig) -> str: + pass + + @abstractmethod + def generate_body(self, config: ModuleConfig) -> str: + pass + + @abstractmethod + def generate_footer(self, config: ModuleConfig) -> str: + pass + +class BaseStrategy(IGenerationStrategy): + def generate_header(self, config: ModuleConfig) -> str: + return f"""#%Module1.0 +set prefix {config.install_root} +set version {config.full_version} +""" + + def generate_body(self, config: ModuleConfig) -> str: + sname = config.software_name.upper().replace('-', '_') + return f""" +setenv {sname}_PATH $prefix +""" + + def generate_footer(self, config: ModuleConfig) -> str: + return "" + +class CompilerStrategy(BaseStrategy): + def generate_body(self, config: ModuleConfig) -> str: + base = super().generate_body(config) + return base + f""" +prepend-path MANPATH $prefix/share/man +""" + + def generate_footer(self, config: ModuleConfig) -> str: + if config.is_pro: + return f""" +prepend-path MODULEPATH {config.attach_module_path} +""" + + +class GCCStrategy(CompilerStrategy): + def generate_body(self, config: ModuleConfig) -> str: + base = super().generate_body(config) + return base + """ +setenv CC gcc +setenv CXX g++ +setenv FC gfortran +""" + +class ClangStrategy(CompilerStrategy): + def generate_body(self, config: ModuleConfig) -> str: + base = super().generate_body(config) + return base + """ +setenv CC clang +setenv CXX clang++ +setenv FC flang +""" + +class MPIStrategy(BaseStrategy): + def generate_body(self, config: ModuleConfig) -> str: + base = super().generate_body(config) + return base + f""" +setenv MPI_HOME $prefix + +setenv OMPI_CC $env(CC) +setenv OMPI_CXX $env(CXX) +setenv OPAL_PREFIX {config.install_root} +setenv PMIX_INSTALL_PREFIX {config.install_root} +setenv MPICC mpicc +setenv MPICXX mpicxx +setenv MPIF90 mpifort +""" + + def generate_footer(self, config: ModuleConfig) -> str: + if config.is_pro: + return f""" +prepend-path MODULEPATH {config.attach_module_path} +""" + + +class HPCKitStrategy(BaseStrategy): + def generate_footer(self, config: ModuleConfig) -> str: + return f""" +prepend-path MODULEPATH {config.install_root}/HPCKit/latest/modulefiles +""" + +class KMLStrategy(BaseStrategy): + def generate_body(self, config: ModuleConfig) -> str: + base = super().generate_body(config) + libs_dir = [ + "$prefix/lib/kblas/nolocking", + "$prefix/lib/kblas/pthread", + "$prefix/lib/kblas/omp", + "$prefix/lib/kblas/locking", + "$prefix/lib/kvml/single", + "$prefix/lib/kvml/multi", + "$prefix/lib/kspblas/single", + "$prefix/lib/kspblas/multi" + ] + return base + f""" +prepend-path LD_LIBRARY_PATH {':'.join(libs_dir)} +""" + diff --git a/src/installService.py b/src/installService.py index 73503a53d28a9cb129ef8034cb0464f1e834f634..5ba955ec557412938d31f8497fd9afb7efd6dd5f 100644 --- a/src/installService.py +++ b/src/installService.py @@ -7,18 +7,22 @@ import fnmatch from enum import Enum from glob import glob from typing import Tuple, Optional, Dict +from pathlib import Path from dataService import DataService from toolService import ToolService from executeService import ExecuteService from jsonService import JSONService from installStrategy import PathStrategyFactory -from installTypes import InstallMode +from installTypes import InstallMode, InstallProfile from versionParser import VersionParser -from detectorService import GCCDetector, ClangDetector,NVCCDetector,ICCDetector -from detectorService import HMPIDetector, OpenMPIDetector, MPICHDetector from softwareFactory import SoftwareFactory +from envFactory import EnvFactory from softwareTypes import SoftwareType +from pathBuilder import PathFactory +from deploymentConfig import DeploymentConfig +from moduleConfig import ModuleConfig +from moduleController import ModulefileEngine class Singleton(type): @@ -34,31 +38,27 @@ class Singleton(type): class InstallService(object,metaclass=Singleton): PACKAGE = 'package' FULL_VERSION='full_version' + INSTALL_JSON = "install.json" + LINK_FILE_NAME = 'linkpathtomodules.sh' def __init__(self): self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() + self.env_factory = EnvFactory() self.software_factory = SoftwareFactory() self.ROOT = os.getcwd() - self.gcc_detector = GCCDetector() - self.clang_detector = ClangDetector() - self.icc_detector = ICCDetector() - self.nvcc_detector = NVCCDetector() - - self.hmpi_detector = HMPIDetector() - self.openmpi_detector = OpenMPIDetector() - self.mpich_detector = MPICHDetector() - self._mode = self._detect_mode() self._init_paths() - self.INSTALL_INFO_PATH = os.path.join(self.SOFTWARE_PATH, "install.json") + self.deployment_config = DeploymentConfig(self._mode, self._paths) + self.path_factory = PathFactory(self.deployment_config) + self.INSTALL_INFO_PATH = os.path.join(self.SOFTWARE_PATH, self.INSTALL_JSON) self._is_first_install = self._set_first_install() self.IS_PRO = self._mode == InstallMode.PRO self.IS_NORMAL = self._mode == InstallMode.NORMAL self.PACKAGE_PATH = os.path.join(self.ROOT, self.PACKAGE) self.dependencies = False - self.LINK_FILE = os.path.join(self.MODULE_FILES,'linkpathtomodules.sh') + self.LINK_FILE = os.path.join(self.MODULE_FILES, self.LINK_FILE_NAME) self._create_install_json(self.INSTALL_INFO_PATH) self._validate_installation() self._prepare_infrastructure() @@ -148,137 +148,6 @@ done ''' self.tool.write_file(self.LINK_FILE, linkfile_content) - def get_mpi_info(self): - mpich_info = self.mpich_detector.detect() - if mpich_info:return mpich_info - hmpi_info = self.hmpi_detector.detect() - if hmpi_info:return hmpi_info - openmpi_info = self.openmpi_detector.detect() - if openmpi_info:return openmpi_info - print("MPI not found, please install MPI first.") - sys.exit() - - def check_compiler_mpi(self, compiler_list, compiler_mpi_info): - no_compiler = ["COM","ANY","MISC","APP"] - is_valid = False - compiler_mpi_info = compiler_mpi_info.upper() - valid_list = [] - for compiler in compiler_list: - valid_list.append(compiler) - valid_list.append(f'{compiler}+MPI') - valid_list += no_compiler - for valid_para in valid_list: - if compiler_mpi_info == valid_para: - is_valid = True - break - if not is_valid: - print(f"compiler or mpi info error, Only {valid_list.join('/').lower()} is supported") - sys.exit() - return compiler_mpi_info - - def get_used_compiler(self, compiler_mpi_info): - return compiler_mpi_info.split('+')[0] - - def get_software_type(self,software_name, compiler_mpi_info): - if self.is_mpi_software(software_name): - return SoftwareType.MPI - if compiler_mpi_info == "COM": - return SoftwareType.COMPILER - elif compiler_mpi_info == "ANY": - return SoftwareType.UTIL - elif compiler_mpi_info == "MISC": - return SoftwareType.MISC - elif compiler_mpi_info == "APP": - return SoftwareType.APP - else: - return SoftwareType.LIB - - def get_compiler_info(self, compilers, compiler_mpi_info): - compiler_info = {"cname":None, "cmversion": None, self.FULL_VERSION: None} - for compiler, info_func in compilers.items(): - if compiler in compiler_mpi_info: - compiler_info = info_func() - return compiler_info - - def get_main_version(self, version): - return version.split('.')[0] - - def is_mpi_software(self, software_name): - mpis = ['hmpi', 'openmpi', 'hpcx', 'mpich'] - for mpi in mpis: - if software_name.startswith(mpi): - return True - return False - - def add_mpi_path(self, software_info, install_path): - if not software_info.use_mpi: - return install_path - mpi_info = self.get_mpi_info() - if mpi_info[self.FULL_VERSION] == None: - print("MPI not found!") - return False - mpi_str = mpi_info["name"]+mpi_info[self.FULL_VERSION] - print("Use MPI: "+mpi_str) - if self.IS_PRO: - install_path = os.path.join(install_path, mpi_str) - elif self.IS_NORMAL: - install_path = f"{install_path}-{mpi_str}" - return install_path - - def get_install_path(self, software_info, env_info): - sversion = software_info.full_version - stype = software_info.software_type - cname = env_info['cname'] - cfullver = env_info[self.FULL_VERSION] - sname = software_info.name - if stype == SoftwareType.MPI: - if self.IS_PRO: - mpi_path = os.path.join(self.MPI_PATH, f"{sname}{sversion}-{cname}{cfullver}", sversion) - if self.IS_NORMAL: - mpi_path = os.path.join(self.MPI_PATH,sname, f"{sversion}-{cname}{cfullver}") - print(f"mpi_path:{mpi_path}") - return mpi_path - if stype == SoftwareType.COMPILER: - install_path = os.path.join(self.COMPILER_PATH, f'{sname}/{sversion}') - elif stype == SoftwareType.UTIL: - install_path = os.path.join(self.UTILS_PATH, f'{sname}/{sversion}') - elif stype == SoftwareType.MISC: - install_path = os.path.join(self.MISC_PATH, f'{sname}/{sversion}') - elif stype == SoftwareType.APP: - install_path = os.path.join(self.APP_PATH, f'{sname}/{sversion}-{cname}{cfullver}') - install_path = self.add_mpi_path(software_info, install_path) - else: - if self.IS_PRO: - # install library - install_path = os.path.join(self.LIBS_PATH, cname+cfullver) - # get mpi name and version - install_path = self.add_mpi_path(software_info, install_path) - install_path = os.path.join(install_path, f'{sname}/{sversion}') - elif self.IS_NORMAL: - install_path = os.path.join(self.LIBS_PATH, f'{sname}/{sversion}-{cname}{cfullver}') - install_path = self.add_mpi_path(software_info, install_path) - return install_path - - def is_contained_mpi(self, compiler_mpi_info): - return "MPI" in compiler_mpi_info - - def get_files(self, abs_path): - file_list = [d for d in glob(abs_path+'/**', recursive=True)] - return file_list - - def add_special_library_path(self, install_path, sname, libs_dir): - prefix = install_path.replace(install_path, "$prefix") - if "kml" in sname: - libs_dir.append(os.path.join(prefix, "lib/kblas/nolocking")) - libs_dir.append(os.path.join(prefix, "lib/kblas/pthread")) - libs_dir.append(os.path.join(prefix, "lib/kblas/omp")) - libs_dir.append(os.path.join(prefix, "lib/kblas/locking")) - libs_dir.append(os.path.join(prefix, "lib/kvml/single")) - libs_dir.append(os.path.join(prefix, "lib/kvml/multi")) - libs_dir.append(os.path.join(prefix, "lib/kspblas/single")) - libs_dir.append(os.path.join(prefix, "lib/kspblas/multi")) - return libs_dir - def get_loaded_modules(self): import subprocess try: @@ -311,76 +180,6 @@ done sys.exit(0) return " ".join(depsmap.values()) - def get_module_file_content(self, install_path, sname, sversion): - module_file_content = '' - if "hpckit" in sname.lower(): - module_file_content = f'''#%Module1.0##################################################################### -prepend-path MODULEPATH {install_path}/HPCKit/latest/modulefiles -''' - return module_file_content - file_list = self.get_files(install_path) - bins_dir_type = ["bin"] - libs_dir_type = ["libs", "lib", "lib64"] - incs_dir_type = ["include"] - bins_dir = [] - libs_dir = [] - incs_dir = [] - compiler_values = '' - bins_str = '' - libs_str = '' - incs_str = '' - opal_prefix = '' - pmix_install_prefix = '' - for file in file_list: - if not os.path.isdir(file): - continue - last_dir = file.split('/')[-1] - if last_dir in bins_dir_type: - bins_dir.append(file.replace(install_path, "$prefix")) - elif last_dir in libs_dir_type: - libs_dir.append(file.replace(install_path, "$prefix")) - elif last_dir in incs_dir_type: - incs_dir.append(file.replace(install_path, "$prefix")) - self.add_special_library_path(install_path, sname, libs_dir) - if len(bins_dir) >= 1: - bins_str = "prepend-path PATH "+':'.join(bins_dir) - if len(libs_dir) >= 1: - libs_str = "prepend-path LD_LIBRARY_PATH "+':'.join(libs_dir) - if len(incs_dir) >= 1: - incs_str = "prepend-path INCLUDE " + ':'.join(incs_dir) - if "bisheng" in sname: - compiler_values = "setenv CC clang \nsetenv CXX clang++ \nsetenv FC flang \nsetenv F77 flang \nsetenv F90 flang " - elif "gcc" in sname: - compiler_values = "setenv CC gcc \nsetenv CXX g++ \nsetenv FC gfortran \nsetenv F77 gfortran \nsetenv F90 gfortran " - elif "hmpi" in sname or "openmpi" in sname: - compiler_values = "setenv CC mpicc \nsetenv CXX mpicxx \nsetenv FC mpifort \nsetenv F77 mpifort \nsetenv F90 mpifort " - if self.is_mpi_software(sname): - opal_prefix = f"setenv OPAL_PREFIX {install_path}" - pmix_install_prefix = f"setenv PMIX_INSTALL_PREFIX {install_path}" - depend_content='' - if self.dependencies: - # complement dependencies - depend_content=''' -foreach dep {%s} { - if { ![is-loaded $dep] } { - module load $dep - } -} -''' % self.dependencies - module_file_content = f'''#%Module1.0##################################################################### -set prefix {install_path} -set version {sversion} -{depend_content} -setenv {sname.upper().replace('-','_')}_PATH {install_path} -{compiler_values} -{opal_prefix} -{pmix_install_prefix} -{bins_str} -{libs_str} -{incs_str} -''' - return module_file_content - def is_installed(self, install_path): #为了兼容老版本，只要安装路径下存在installed也算做已安装 installed_file_path = os.path.join(install_path, "installed") @@ -388,89 +187,6 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} return True return self.json.get(install_path) - def gen_module_file(self, install_path, software_info, env_info): - sname = software_info.name - sversion = software_info.full_version - stype = software_info.software_type - cname = env_info['cname'] - cfullversion = env_info[self.FULL_VERSION] - module_file_content = self.get_module_file_content(install_path, sname, sversion) - if not self.is_installed(install_path): - return '' - # if install_path is empty, The module file should not generated. - if len(os.listdir(install_path)) == 0: - print('module file did not generated because no file generated under install path') - return '' - if stype == SoftwareType.MPI: - compiler_str = cname + cfullversion - software_str = sname + sversion - if self.IS_PRO: - module_path = os.path.join(self.MODULE_DEPS_PATH, compiler_str ,sname) - attach_module_path = os.path.join(self.MODULE_DEPS_PATH, compiler_str+'-'+software_str) - self.tool.mkdirs(attach_module_path) - module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" - print(f'attach module file {attach_module_path} successfully generated.') - elif self.IS_NORMAL: - module_path = os.path.join(self.MODULE_COMPILER_PATH, sname, f'{sversion}-{compiler_str}') - else: - if stype == SoftwareType.COMPILER: - software_str = sname + sversion - if self.IS_PRO: - module_path = os.path.join(self.MODULE_FILES, sname) - attach_module_path = os.path.join(self.MODULE_DEPS_PATH, software_str) - self.tool.mkdirs(attach_module_path) - module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" - print(f'attach module file {attach_module_path} successfully generated.') - elif self.IS_NORMAL: - module_path = os.path.join(self.MODULE_COMPILER_PATH, sname, sversion) - elif stype == SoftwareType.UTIL: - if self.IS_PRO: - module_path = os.path.join(self.MODULE_FILES, sname) - elif self.IS_NORMAL: - module_path = os.path.join(self.MODULE_UTIL_PATH, sname, sversion) - elif stype == SoftwareType.MISC: - if self.IS_PRO: - module_path = os.path.join(self.MODULE_FILES, sname) - elif self.IS_NORMAL: - module_path = os.path.join(self.MODULE_MISC_PATH, sname, sversion) - else: - if self.IS_NORMAL: - if stype == SoftwareType.APP: - BASE_PATH = self.MODULE_APP_PATH - else: - BASE_PATH = self.MODULE_LIB_PATH - elif self.IS_PRO: - if stype == SoftwareType.APP: - BASE_PATH = self.MODULE_APP_PATH - else: - BASE_PATH = self.MODULE_DEPS_PATH - compiler_str = cname + cfullversion - if software_info.use_mpi: - mpi_info = self.get_mpi_info() - mpi_str = mpi_info['name'] + mpi_info[self.FULL_VERSION] - if self.IS_PRO: - module_path = os.path.join(self.BASE_PATH, f"{compiler_str}-{mpi_str}" ,sname) - if self.IS_NORMAL: - module_path = os.path.join(BASE_PATH, sname, f"{sversion}-{compiler_str}-{mpi_str}") - else: - if self.IS_PRO: - module_path = os.path.join(self.BASE_PATH, compiler_str, sname) - elif self.IS_NORMAL: - module_path = os.path.join(BASE_PATH, sname, f"{sversion}-{compiler_str}") - if self.IS_PRO: - self.tool.mkdirs(module_path) - module_file = os.path.join(module_path, sversion) - elif self.IS_NORMAL: - self.tool.mkdirs(os.path.dirname(module_path)) - module_file = module_path - self.tool.write_file(module_file, module_file_content) - print(f"module file {module_file} successfully generated") - row = self.json.get(install_path) - row["module_path"] = module_file - self.json.set(install_path, row, True) - #更新linkfile - if self.IS_NORMAL:self.update_modules() - def update_modules(self): upd_cmd = f''' cd {self.MODULE_FILES} @@ -543,54 +259,79 @@ chmod +x {install_script} software_dict['version'] = software_info.full_version software_dict['module_path'] = '' self.json.set(install_path, software_dict, True) - - def remove_prefix(self, software_path): - if software_path.startswith('package/') or software_path.startswith('./'): - software_path = software_path.replace('./', '', 1) - software_path = software_path.replace('package/', '', 1) - return software_path + def _write_modulefile(self, install_path, module_path, content): + base_path = os.path.dirname(module_path) + self.tool.mkdirs(base_path) + self.tool.write_file(module_path, content) + print(f"module file {module_path} successfully generated") + row = self.json.get(install_path) + row["module_path"] = module_path + self.json.set(install_path, row, True) + #更新linkfile + if self.IS_NORMAL:self.update_modules() + + def _get_attach_module_path(self, software_info, env_info): + if self.IS_NORMAL: + return "" + stype = software_info.software_type + attach_module_path = "" + if stype == SoftwareType.MPI: + attach_module_path = os.path.join(self.MODULE_DEPS_PATH, f"{env_info.compiler_name}{env_info.compiler_version}-{software_info.name}{software_info.full_version}") + self.tool.mkdirs(attach_module_path) + elif stype == SoftwareType.COMPILER: + attach_module_path = os.path.join(self.MODULE_DEPS_PATH, f"{software_info.name}{software_info.full_version}") + self.tool.mkdirs(attach_module_path) + return attach_module_path + + def gen_module_file(self, install_path, software_info, env_info): + stype = software_info.software_type + # 构建配置对象 + deps = self.dependencies.split() if self.dependencies else [] + attach_mod_path = self._get_attach_module_path(software_info, env_info) + config = ModuleConfig( + is_pro = self.IS_PRO, + install_root=Path(install_path), + software_name=software_info.name, + full_version=software_info.full_version, + category=stype, + dependencies=deps, + attach_module_path=attach_mod_path + ) + + # 生成模块文件 + engine = ModulefileEngine() + module_file_content = engine.generate(config) + module_path = self.path_factory.get_module_builder(stype).build_path(software_info, env_info) + module_path = str(module_path) + self._write_modulefile(install_path, module_path, module_file_content) + def install(self, install_args, isapp = False): software_path = install_args[0] compiler_mpi_info = install_args[1] other_args = install_args[2:] self.tool.prt_content("INSTALL " + software_path) - compilers = {"GCC":self.gcc_detector.detect, "CLANG":self.clang_detector.detect, - "NVC":self.nvcc_detector.detect, "ICC":self.icc_detector.detect, - "BISHENG": self.clang_detector.detect} - compiler_mpi_info = self.check_compiler_mpi(compilers.keys(), compiler_mpi_info) software_info = self.software_factory.create_profile(software_path, compiler_mpi_info, isapp) + env_info = self.env_factory.create_profile(software_info, compiler_mpi_info) stype = software_info.software_type - # get compiler name and version - env_info = self.get_compiler_info(compilers, compiler_mpi_info) - if stype == SoftwareType.LIB or stype == SoftwareType.MPI or stype == SoftwareType.APP: - cmversion = env_info['cmversion'] - cfullver = env_info[self.FULL_VERSION] - if cmversion == None: - print(f"The specified {software_info.compiler_name} Compiler not found!") - return False - else: - print(f"Use Compiler: {env_info['cname']} {cfullver}") - # get install path - install_path = self.get_install_path(software_info, env_info) + install_path = self.path_factory.get_builder(stype).build_path(software_info, env_info) + install_path = str(install_path) if not install_path: return - else: - self.tool.mkdirs(install_path) - if isapp: - return { - "install_path": install_path, - "software_info":software_info, - "env_info":env_info - } + self.tool.mkdirs(install_path) + install_profile = InstallProfile( + install_path = install_path, + software_info = software_info, + env_info = env_info + ) + if isapp: return install_profile # get install script self.install_package(software_info.install_script_path, install_path, other_args) # add install info self.add_install_info(software_info, install_path) - # gen module file - self.gen_module_file( install_path, software_info, env_info) + self.gen_module_file(install_path, software_info, env_info) def install_depend(self): depend_file = self.ds.get_depend_file() diff --git a/src/installTypes.py b/src/installTypes.py index 3bbf56a0d31490c1f4ccdd84cd564ae59b6d0c37..9eb80c080b5dcd3d7e9949272e3239d034d533ed 100644 --- a/src/installTypes.py +++ b/src/installTypes.py @@ -1,18 +1,17 @@ -# types.py +from dataclasses import dataclass from enum import Enum from typing import Tuple, Optional, Dict +from softwareTypes import SoftwareProfile, EnvironmentProfile class InstallMode(Enum): PRO = "0" NORMAL = "1" -class SoftwareCategory(Enum): - COMPILER = 1 - MPI = 2 - UTIL = 3 - LIB = 4 - MISC = 5 - APP = 6 +@dataclass +class InstallProfile: + install_path: str = "" + software_info: SoftwareProfile = None + env_info: EnvironmentProfile = None VersionInfo = Tuple[str, str] # (major_version, full_version) diff --git a/src/jsonService.py b/src/jsonService.py index 4f7e05de384ebf3f8bc68057367ab19fd01a2e7c..7cb5117eba98ebd30ae4c444f284a252286997ac 100644 --- a/src/jsonService.py +++ b/src/jsonService.py @@ -67,11 +67,16 @@ class JSONService: :param auto_save: 是否自动保存到文件 :raises TypeError: 当键名不是字符串时抛出 """ + key = self.path_to_str(key) if not isinstance(key, str): raise TypeError("键名必须是字符串类型") self._data[key] = value if auto_save: self.save() + + def path_to_str(self, obj) -> str: + """若对象是 Path 类型则返回字符串，否则返回原对象""" + return str(obj) if isinstance(obj, Path) else obj def delete(self, key: str, auto_save: bool = False) -> bool: """ diff --git a/src/moduleConfig.py b/src/moduleConfig.py new file mode 100644 index 0000000000000000000000000000000000000000..61beded58f66bcfb265c2690cc74656526c13c6a --- /dev/null +++ b/src/moduleConfig.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from pathlib import Path +from softwareTypes import SoftwareType +from typing import Tuple, Optional, Dict, List + +@dataclass +class ModuleConfig: + is_pro:bool = True + install_root: Path = None + software_name: str = "" + full_version: str = "" + category: SoftwareType = None + dependencies: List[str] = None + attach_module_path: str = "" + diff --git a/src/moduleController.py b/src/moduleController.py new file mode 100644 index 0000000000000000000000000000000000000000..a4610b47062306e00ab4c85f2b4fdf3e136d4b16 --- /dev/null +++ b/src/moduleController.py @@ -0,0 +1,74 @@ +from pathlib import Path +from generationStrategy import IGenerationStrategy, BaseStrategy, GCCStrategy, ClangStrategy, MPIStrategy, HPCKitStrategy, KMLStrategy +from pathManager import PathOrganizer +from softwareTypes import SoftwareType +from moduleConfig import ModuleConfig +from dependencyManager import DependencyResolver + +class ModulefileEngine: + STRATEGY_MAP = { + "gcc": GCCStrategy(), + "kgcc": GCCStrategy(), + "clang": ClangStrategy(), + "bisheng": ClangStrategy(), + "openmpi": MPIStrategy(), + "hmpi": MPIStrategy(), + "hpckit": HPCKitStrategy(), + "kml-gcc": KMLStrategy(), + "kml-bisheng": KMLStrategy() + } + + def __init__(self, strategy: IGenerationStrategy = None): + self.strategy = strategy or BaseStrategy() + + def generate(self, config: ModuleConfig) -> str: + self.strategy = self.STRATEGY_MAP.get(config.software_name, BaseStrategy()) + sections = [ + self.strategy.generate_header(config), + self.strategy.generate_body(config), + self._generate_dependencies(config), + self._generate_paths(config), + self.strategy.generate_footer(config) + ] + return "\n".join(filter(None, sections)) + + def _generate_paths2(self, config: ModuleConfig) -> str: + if config.software_name == "hpckit": + return "" + path_config = PathOrganizer.auto_discover(config.install_root) + return f""" +prepend-path PATH {PathOrganizer.format_paths(path_config['bins'])} +prepend-path LD_LIBRARY_PATH {PathOrganizer.format_paths(path_config['libs'])} +prepend-path C_INCLUDE_PATH {PathOrganizer.format_paths(path_config['includes'])} +prepend-path INCLUDE {PathOrganizer.format_paths(path_config['includes'])} +""" + + def _generate_paths(self, config: ModuleConfig) -> str: + if config.software_name == "hpckit": + return "" + + path_config = PathOrganizer.auto_discover(config.install_root) + path_commands = [] + + # 环境变量与路径配置的映射关系 + env_mapping = { + 'PATH': ('bins', lambda x: x), + 'LD_LIBRARY_PATH': ('libs', lambda x: x), + 'C_INCLUDE_PATH': ('includes', lambda x: x), + 'INCLUDE': ('includes', lambda x: x) + } + + # 动态生成有效路径指令 + for env_var, (config_key, formatter) in env_mapping.items(): + paths = path_config.get(config_key, []) + if not paths: + continue + + formatted_paths = PathOrganizer.format_paths(formatter(paths)) + path_commands.append(f"prepend-path {env_var} {formatted_paths}") + + return "\n".join(path_commands) if path_commands else "" + + def _generate_dependencies(self, config: ModuleConfig) -> str: + return DependencyResolver(config.dependencies).generate_load_commands() + diff --git a/src/pathBuilder.py b/src/pathBuilder.py index e6b63a81af7363c312fb9e982484498cefb03fe4..5e82a271f3d7577c17491e0c428ad9f372ac6760 100644 --- a/src/pathBuilder.py +++ b/src/pathBuilder.py @@ -1,16 +1,30 @@ from abc import ABC, abstractmethod from pathlib import Path -from softwareTypes import SoftwareProfile, EnvironmentProfile +from softwareTypes import SoftwareType, SoftwareProfile, EnvironmentProfile from collections import defaultdict +from deploymentConfig import DeploymentConfig +from collections import defaultdict +import os class PathBuilder(ABC): - def __init__(self, base_config: 'DeploymentConfig'): + def __init__(self, base_config: DeploymentConfig): self.config = base_config @abstractmethod def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: pass + def add_mpi_path(self, software_info, env_info, install_path): + if not software_info.use_mpi: + return install_path + mpi_str = env_info.mpi_name + env_info.mpi_version + print("Use MPI: "+mpi_str) + if self.config.is_pro: + install_path = os.path.join(install_path, mpi_str) + else: + install_path = f"{install_path}-{mpi_str}" + return install_path + class MpiPathBuilder(PathBuilder): def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: folder_pattern = ( @@ -21,33 +35,98 @@ class MpiPathBuilder(PathBuilder): def _construct_mpi_path(self, pattern: str, software: SoftwareProfile, env: EnvironmentProfile) -> Path: path_vars = { - 'name': software.name, - 'version': software.full_version, - 'compiler': env.compiler_name, - 'compiler_ver': env.compiler_version - } + 'name': software.name, + 'version': software.full_version, + 'compiler': env.compiler_name, + 'compiler_ver': env.compiler_version + } return self.config.mpi_base / pattern.format(**path_vars) class CompilerPathBuilder(PathBuilder): def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: - return self.config.compiler_base / f"{software.name}/{software.version}" + return self.config.compiler_base / f"{software.name}/{software.full_version}" + +class UtilPathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + util_path = self.config.util_base / f"{software.name}/{software.full_version}" + return util_path.absolute() + +class MiscPathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + return self.config.misc_base / f"{software.name}/{software.full_version}" class LibPathBuilder(PathBuilder): def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: - base_path = self.config.pro_lib_base if self.config.is_pro else self.config.normal_lib_base - path_segments = [ - f"{env.compiler_name}{env.compiler_version}", - *([f"mpi{env.mpi_version}"] if software.requires_mpi else []), - f"{software.name}/{software.version}" - ] - return base_path.joinpath(*path_segments) + base_path = self.config.lib_base + if self.config.is_pro: + lib_path = os.path.join(base_path, f"{env.compiler_name}{env.compiler_version}") + lib_path = self.add_mpi_path(software, env, lib_path) + lib_path = os.path.join(lib_path, f"{software.name}/{software.full_version}") + else: + lib_path = os.path.join(base_path, f"{software.name}/{software.full_version}-{env.compiler_name}{env.compiler_version}") + lib_path = self.add_mpi_path(software, env, lib_path) + return lib_path -from collections import defaultdict +class AppPathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + base_path = self.config.app_base + app_path = os.path.join(base_path, f"{software.name}/{software.full_version}-{env.compiler_name}{env.compiler_version}") + app_path = self.add_mpi_path(software, env, app_path) + return app_path + +class MpiModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + folder_pattern = ( + self.config.pro_module_path_pattern if self.config.is_pro + else self.config.normal_path_pattern + ) + return self._construct_mpi_path(folder_pattern, software, env) + + def _construct_mpi_path(self, pattern: str, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + path_vars = { + 'name': software.name, + 'version': software.full_version, + 'compiler': env.compiler_name, + 'compiler_ver': env.compiler_version + } + return self.config.mpi_module_base / pattern.format(**path_vars) + +class CompilerModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + return self.config.compiler_module_base / f"{software.name}/{software.full_version}" + +class UtilModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + return self.config.util_module_base / f"{software.name}/{software.full_version}" + +class MiscModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + return self.config.misc_module_base / f"{software.name}/{software.full_version}" + +class LibModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + base_path = self.config.lib_module_base + if self.config.is_pro: + if software.use_mpi: + lib_path = os.path.join(base_path, f"{env.compiler_name}{env.compiler_version}-{env.mpi_name}{env.mpi_version}", software.name, software.full_version) + else: + lib_path = os.path.join(base_path, f"{env.compiler_name}{env.compiler_version}", software.name, software.full_version) + else: + lib_path = os.path.join(base_path, f"{software.name}/{software.full_version}-{env.compiler_name}{env.compiler_version}") + lib_path = self.add_mpi_path(software, env, lib_path) + return lib_path + +class AppModulePathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + base_path = self.config.app_module_base + app_path = os.path.join(base_path, f"{software.name}/{software.full_version}-{env.compiler_name}{env.compiler_version}") + app_path = self.add_mpi_path(software, env, app_path) + return app_path class PathFactory: _builder_registry = defaultdict(lambda: None) - - def __init__(self, config: 'DeploymentConfig'): + _builder_module_registry = defaultdict(lambda: None) + def __init__(self, config: DeploymentConfig): self.config = config self._initialize_builders() @@ -60,10 +139,23 @@ class PathFactory: SoftwareType.UTIL: UtilPathBuilder(self.config), SoftwareType.MISC: MiscPathBuilder(self.config) }) - + self._builder_module_registry.update({ + SoftwareType.MPI: MpiModulePathBuilder(self.config), + SoftwareType.COMPILER: CompilerModulePathBuilder(self.config), + SoftwareType.LIB: LibModulePathBuilder(self.config), + SoftwareType.APP: AppModulePathBuilder(self.config), + SoftwareType.UTIL: UtilModulePathBuilder(self.config), + SoftwareType.MISC: MiscModulePathBuilder(self.config) + }) def get_builder(self, software_type: SoftwareType) -> PathBuilder: + builder = self._builder_registry[software_type] + if builder: + return builder + raise ValueError(f"No path builder registered for {software_type}") - if builder := self._builder_registry[software_type]: + def get_module_builder(self, software_type: SoftwareType) -> PathBuilder: + builder = self._builder_module_registry[software_type] + if builder: return builder raise ValueError(f"No path builder registered for {software_type}") diff --git a/src/pathManager.py b/src/pathManager.py new file mode 100644 index 0000000000000000000000000000000000000000..e5485b6e32e78faf8daacb2c74010ebf4ae90696 --- /dev/null +++ b/src/pathManager.py @@ -0,0 +1,19 @@ +from pathlib import Path + +class PathOrganizer: + install_path = None + @staticmethod + def auto_discover(install_path: Path): + PathOrganizer.install_path = install_path + return { + "bins": sorted([p for p in install_path.glob("**/bin") if p.is_dir()]), + "libs": sorted([p for p in install_path.glob("**/lib*") if p.is_dir()]), + "includes": sorted([p for p in install_path.glob("**/include*") if p.is_dir()]) + } + + @staticmethod + def format_paths(paths) -> str: + if PathOrganizer.install_path is None: + raise ValueError("install_path is not set. Please call auto_discover first.") + return ":".join(f"$prefix/{p.relative_to(PathOrganizer.install_path)}" for p in paths) + diff --git a/src/runService.py b/src/runService.py index 5e3f634a58af94114084635005f01cfd7b100444..7b8680256ab4faa9a738cb33e3074d9dfdc18c45 100644 --- a/src/runService.py +++ b/src/runService.py @@ -5,11 +5,13 @@ import sys from toolService import ToolService from executeService import ExecuteService from dataService import DataService +from envService import EnvService from commandBuilder import CommandBuilder class RunService: def __init__(self): self.ds = DataService() + self.env_service = EnvService() self.exe = ExecuteService() self.tool = ToolService() self.command = CommandBuilder() # 注入命令生成组件 @@ -32,13 +34,16 @@ class RunService: print(f"start run {self.ds.get_app_name()}") nodes = int(self.ds.get_run_cmd('nodes')) self.gen_hostfile(nodes) + self.env_service.env() run_cmd = self.command.full_run() self.exe.exec_raw(run_cmd) def batch_run(self): + self.env_service.env() batch_cmd = self.command.batch_run() self.exe.exec_raw(run_cmd) def job_run(self,num): + self.env_service.env() job_cmd = self.command.job_run(num) self.exe.exec_raw(job_cmd) diff --git a/src/softwareFactory.py b/src/softwareFactory.py index 166a801beeebd41cd25dda5546a1f7f3292e109f..3b8973bd76233173a9a070687387da47e4a1ce20 100644 --- a/src/softwareFactory.py +++ b/src/softwareFactory.py @@ -14,18 +14,19 @@ class SoftwareFactory: 'default': DefaultStrategy() } - def create_profile(self, software_path: str, compiler_info: str, isapp=False) -> SoftwareProfile: + def create_profile(self, software_path: str, compiler_mpi_info: str, isapp=False) -> SoftwareProfile: """创建软件信息对象的统一入口""" + compiler_mpi_info = compiler_mpi_info.upper() (software_name, version, suffix) = self._parse_name_and_version(software_path) - software_type = self._determine_type(software_name, compiler_info, isapp) + software_type = self._determine_type(software_name, compiler_mpi_info, isapp) profile = SoftwareProfile( name=self._handle_suffix(software_name, suffix), full_version=version, major_version=self._parse_major_version(version), software_type=software_type, suffix=suffix, - use_mpi='+MPI' in compiler_info, - compiler_name=compiler_info.split('+')[0] if '+' in compiler_info else compiler_info, + use_mpi='+MPI' in compiler_mpi_info, + compiler_name=compiler_mpi_info.split('+')[0] if '+' in compiler_mpi_info else compiler_mpi_info, install_script_path=self._get_install_script_path(software_name, version, software_type, suffix), ) return profile diff --git a/src/softwareTypes.py b/src/softwareTypes.py index bfee1931f36bb93c4e43ac5a6eadccd467467791..120424e89638964c42e84a0aa19678762a28885a 100644 --- a/src/softwareTypes.py +++ b/src/softwareTypes.py @@ -25,3 +25,6 @@ class EnvironmentProfile: compiler_name: str compiler_version: str compiler_major_version: str + mpi_name: str = None + mpi_version: str = None + mpi_major_version: str = None diff --git a/templates/top50/gromacs.2023.3.config b/templates/top50/gromacs.2023.3.config new file mode 100644 index 0000000000000000000000000000000000000000..40abd5fada527dd334adeb9c98b4ce4843e4ce19 --- /dev/null +++ b/templates/top50/gromacs.2023.3.config @@ -0,0 +1,89 @@ +[SERVER] +11.11.11.11 + +[DOWNLOAD] +gromacs/2023.3 http://ftp.gromacs.org/pub/gromacs/gromacs-2023.3.tar.gz +Testcase https://repository.prace-ri.eu/ueabs/GROMACS/2.2/GROMACS_TestCaseC.tar.xz + +[DEPENDENCY] +set -e +set -x +module purge +module use $JARVIS_MODULES/modules +./jarvis -install cmake/3.28.2 any +./jarvis -install hpckit/2025.3.30 any +module load hpckit/2025.3.30 +module load bisheng/compiler4.2.0/bishengmodule +./jarvis -install openblas/0.3.14 any +module load bisheng/hmpi25.0.0/hmpi +export CC=mpicc CXX=mpicxx FC=mpifort +./jarvis -install fftw/3.3.10 clang+mpi +#解压源码 +cd $JARVIS_DEV +tar -xvf ${JARVIS_DOWNLOAD}/gromacs-2023.3.tar.gz +#解压算例 +CASE_DIR=${JARVIS_JOBSCRIPT}/gromacs/2023.3 +if [ ! -d $CASE_DIR ]; then + mkdir -p $CASE_DIR + cd $CASE_DIR + tar -xf ${JARVIS_DOWNLOAD}/GROMACS_TestCaseC.tar.xz +fi + +[ENV] +module purge +module use $JARVIS_MODULES/modules +module load cmake/3.28.2 +module load hpckit/2025.3.30 +#低版本GCC会报错 +module add gcc/compiler12.3.1/gccmodule +module load bisheng/compiler4.2.0/bishengmodule +module load bisheng/hmpi25.0.0/hmpi +module load openblas/0.3.14 +module load fftw/3.3.10-bisheng4.2.0-hmpi25.0.0 +compile_dir=$(dirname $(dirname $(which clang))) +export gcc_compile_dir=$(dirname $(dirname $(which gcc))) +export gromacs_dir=$JARVIS_DEV/gromacs-2023.3 + +[APP] +app_name = gromacs +app_version = 2023.3 +compiler = bisheng+mpi +build_dir = $JARVIS_DEV/gromacs-2023.3 +binary_dir = $JARVIS_DEV/gromacs-2023.3/build/bin +case_dir = ${JARVIS_JOBSCRIPT}/gromacs/2023.3/GROMACS_TestCaseC + +[BUILD] +cmake --version +mkdir -p build +cd build +FLAGS="-mcpu=linxicore9100 -O3 -ffast-math -mllvm --aarch64-sched-inline-asm=false -mllvm -unroll-threshold-aggressive=600" +LD_FLAGS="-mcpu=linxicore9100 -O3 -L${gcc_compile_dir}/lib64 -lgfortran -lbsmath -lflang -L${compile_dir}/lib/jemalloc-64kbpage -ljemalloc" + +CFLAGS=$FLAGS CXXFLAGS=$FLAGS LDFLAGS=$LD_FLAGS CC=mpicc CXX=mpicxx FC=mpifort cmake -DCMAKE_INSTALL_PREFIX=$1 -DBUILD_SHARED_LIBS=on -DBUILD_TESTING=on -DREGRESSIONTEST_DOWNLOAD=off -DGMX_BUILD_OWN_FFTW=off -DGMX_SIMD=ARM_SVE -DGMX_SIMD_ARM_SVE_LENGTH=256 -DGMX_DOUBLE=off -DGMX_EXTERNAL_BLAS=on -DGMX_EXTERNAL_LAPACK=on -DGMX_FFT_LIBRARY=fftw3 -DGMX_BLAS_USER=${OPENBLAS_PATH}/lib/libopenblas.a -DGMX_LAPACK_USER=${OPENBLAS_PATH}/lib/libopenblas.a -DFFTWF_LIBRARY=$FFTW_PATH/lib/libfftw3f.so -DFFTWF_INCLUDE_DIR=$FFTW_PATH/include -DGMX_X11=off -DGMX_OPENMP=off -DGMX_MPI=on -DGMX_THREAD_MPI=off -DGMX_CYCLE_SUBCOUNTERS=off ../ + +make -j40 V=1 +make install + +[CLEAN] +rm -rf build + +[RUN] +run = mpirun -np 574 --allow-run-as-root -x UCX_TLS=sm --bind-to cpulist:ordered -mca pml ucx -mca btl ^vader,tcp,openib,uct +binary = gmx_mpi mdrun -dlb yes -v -nsteps 4000 -noconfout -pin on -pinoffset 0 -ntomp 1 -npme 112 -g md_sve_0229-ucpg-bisheng-2P.log -s stmv.28M.tpr +nodes = 1 + +[JOB] +#!/bin/bash +#DSUB -n gmx +#DSUB --job_type cosched +#DSUB -N 1 +#DSUB -R "cpu=128" +#DSUB -o gmx_%J.log +#DSUB -e gmx_err_%J.log +#DSUB -T '2h' + +export HOSTFILE=hostfile.gmx +rm -f $HOSTFILE +touch $HOSTFILE +cat ${CCSCHEDULER_ALLOC_FILE} | sort > $HOSTFILE +mpirun -np 128 --hostfile $HOSTFILE -x UCX_TLS=sm --bind-to cpulist:ordered -mca pml ucx -mca btl ^vader,tcp,openib,uct ${gromacs_dir}/build/bin/gmx_mpi mdrun -dlb yes -v -nsteps 4000 -noconfout -pin on -pinoffset 0 -ntomp 1 -npme 8 -g 3-5-pme8-128C.log -s stmv.28M.tpr diff --git a/templates/top50/qe.6.4.1.config b/templates/top50/qe.6.4.1.config new file mode 100644 index 0000000000000000000000000000000000000000..a2a79cc0ef23f1cd6fa7810ccf799cc52163e1b2 --- /dev/null +++ b/templates/top50/qe.6.4.1.config @@ -0,0 +1,77 @@ +[SERVER] +11.11.11.11 + +[DOWNLOAD] +qe/6.4 $JARVIS_PROXY/QEF/q-e/archive/refs/tags/qe-6.4.1.tar.gz q-e-qe-6.4.1.tar.gz + +[DEPENDENCY] +set -e +set -x +module purge +module use $JARVIS_MODULES/modules +./jarvis -install hpckit/2025.3.30 any +module load hpckit/2025.3.30 +module load bisheng/compiler4.2.0/bishengmodule +module load bisheng/hmpi25.0.0/hmpi +cd $JARVIS_DEV +tar -xzvf ${JARVIS_DOWNLOAD}/q-e-qe-6.4.1.tar.gz + +[ENV] +module purge +module use $JARVIS_MODULES/modules +module load hpckit/2025.3.30 +module load bisheng/compiler4.2.0/bishengmodule +module load bisheng/hmpi25.0.0/hmpi +export QE_ROOT=${JARVIS_DEV}/q-e-qe-6.4.1/ + +[APP] +app_name = QE +app_version = 6.4.1 +compiler = bisheng+mpi +build_dir = ${JARVIS_DEV}/q-e-qe-6.4.1/ +binary_dir = ${JARVIS_DEV}/q-e-qe-6.4.1/bin/ +case_dir = ${JARVIS_JOBSCRIPT}/QE/6.4.1/ + +[BUILD] +set -e +set -x +#$PATCH_TOOL $JARVIS_TPL/top50/qe/6.4.1/opt.patch +./configure F90=flang F77=flang MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp --prefix=$1 +make -j pw +make install + +[CLEAN] +make clean + +[RUN] +run = mpirun --allow-run-as-root -x OMP_NUM_THREADS=1 -np 8 +binary = pw.x -input test_3.in +nodes = 1 + +[BATCH] +mpirun --allow-run-as-root -x OMP_NUM_THREADS=1 -np 8 $QE_ROOT/bin/pw.x -input test_3.in + +[JOB] +#!/bin/sh +#DSUB -n qe_test +#DSUB --job_type cosched:hmpi +#DSUB -A root.default +#DSUB -q root.default +#DSUB -N 1 +#DSUB -R cpu=128 +#DSUB -oo qe.%J.out +#DSUB -eo qe.%J.err + +##set runtime environment variables +module load QE +ulimit -s unlimited +ulimit -c unlimited +echo "----HOSTFILE generated---" +cat $CCS_HOST_FILE +echo "-------------------------" +EXEC_CMD="time -p mpirun $CCS_MPI_OPTIONS -n 32 -x OMP_NUM_THREADS=1 -mca pml ucx -mca btl ^vader,tcp,openib,uct -x UCX_TLS=self,sm,rc pw.x -input test_3.in" +echo "$EXEC_CMD" +date +$EXEC_CMD +date + diff --git a/test-blast.sh b/test/test-blast.sh similarity index 100% rename from test-blast.sh rename to test/test-blast.sh