diff --git a/init.sh b/init.sh index ca490e5f651ce7663c8b305ef6910a207da6674f..862035a86c584a3a43ac814b7f560ae86851bf97 100644 --- a/init.sh +++ b/init.sh @@ -31,7 +31,7 @@ elif [ "$JARVIS_MODE" -eq 1 ]; then export JARVIS_MISC=${JARVIS_SOFT_ROOT}/soft/misc export JARVIS_APP=${JARVIS_SOFT_ROOT}/soft/app export JARVIS_MODULES_LIB=${JARVIS_SOFT_ROOT}/modulefile/lib - export JARVIS_MODULES_TOOL=${JARVIS_SOFT_ROOT}/modulefile/tool + export JARVIS_MODULES_UTIL=${JARVIS_SOFT_ROOT}/modulefile/tool export JARVIS_MODULES_COMPILER=${JARVIS_SOFT_ROOT}/modulefile/compiler export JARVIS_MODULES_MPI=${JARVIS_SOFT_ROOT}/modulefile/mpi export JARVIS_MODULES_LIB=${JARVIS_SOFT_ROOT}/modulefile/lib diff --git a/package/zlib/1.2.11/install.sh b/package/zlib/1.2.11/install.sh index 8387d3dcda6de44bbe4b6d5b142189c5c5ba3107..39c232ffcccb340aec5a1edf8e28f46a0a86dc1c 100755 --- a/package/zlib/1.2.11/install.sh +++ b/package/zlib/1.2.11/install.sh @@ -1,7 +1,6 @@ #!/bin/bash set -x set -e -export DEPENDENCIES="openblas" zlib_ver='1.2.11' . ${DOWNLOAD_TOOL} -u https://zlib.net/fossils/zlib-${zlib_ver}.tar.gz cd ${JARVIS_TMP} diff --git a/src/installService.py b/src/installService.py index efc81cce88c03c8306679405d160240dd81e9f30..73503a53d28a9cb129ef8034cb0464f1e834f634 100644 --- a/src/installService.py +++ b/src/installService.py @@ -17,6 +17,8 @@ from installTypes import InstallMode from versionParser import VersionParser from detectorService import GCCDetector, ClangDetector,NVCCDetector,ICCDetector from detectorService import HMPIDetector, OpenMPIDetector, MPICHDetector +from softwareFactory import SoftwareFactory +from softwareTypes import SoftwareType class Singleton(type): @@ -29,15 +31,6 @@ class Singleton(type): self._instance = super(Singleton,self).__call__(*args, **kwargs) return self._instance - -class SType(Enum): - COMPILER = 1 - MPI = 2 - UTIL = 3 - LIB = 4 - MISC = 5 - APP = 6 - class InstallService(object,metaclass=Singleton): PACKAGE = 'package' FULL_VERSION='full_version' @@ -45,6 +38,7 @@ class InstallService(object,metaclass=Singleton): self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() + self.software_factory = SoftwareFactory() self.ROOT = os.getcwd() self.gcc_detector = GCCDetector() @@ -57,7 +51,7 @@ class InstallService(object,metaclass=Singleton): self.mpich_detector = MPICHDetector() self._mode = self._detect_mode() - self._paths = self._init_paths() + self._init_paths() self.INSTALL_INFO_PATH = os.path.join(self.SOFTWARE_PATH, "install.json") self._is_first_install = self._set_first_install() self.IS_PRO = self._mode == InstallMode.PRO @@ -73,7 +67,7 @@ class InstallService(object,metaclass=Singleton): if self._is_first_install: self.tool.write_file(filename, "{}") self.json = JSONService(filename) - self.json.set("MODE", self._mode, True) + self.json.set("MODE", self._mode.value, True) return # 处理老的目录结构 self.json = JSONService(filename) @@ -164,13 +158,6 @@ done print("MPI not found, please install MPI first.") sys.exit() - def check_software_path(self, software_path): - abs_software_path = os.path.join(self.PACKAGE_PATH, software_path) - if not os.path.exists(abs_software_path): - print(f"{software_path} not exist, are you sure the software lies in package dir?") - return False - return abs_software_path - def check_compiler_mpi(self, compiler_list, compiler_mpi_info): no_compiler = ["COM","ANY","MISC","APP"] is_valid = False @@ -186,7 +173,7 @@ done break if not is_valid: print(f"compiler or mpi info error, Only {valid_list.join('/').lower()} is supported") - return False + sys.exit() return compiler_mpi_info def get_used_compiler(self, compiler_mpi_info): @@ -194,43 +181,17 @@ done def get_software_type(self,software_name, compiler_mpi_info): if self.is_mpi_software(software_name): - return SType.MPI + return SoftwareType.MPI if compiler_mpi_info == "COM": - return SType.COMPILER + return SoftwareType.COMPILER elif compiler_mpi_info == "ANY": - return SType.UTIL + return SoftwareType.UTIL elif compiler_mpi_info == "MISC": - return SType.MISC + return SoftwareType.MISC elif compiler_mpi_info == "APP": - return SType.APP - else: - return SType.LIB - - def get_suffix(self, software_info_list): - if len(software_info_list) >= 3: - return software_info_list[2] - return "" - - def get_software_info(self, software_path, compiler_mpi_info, isapp = False): - software_info_list = software_path.split('/') - software_name = software_info_list[0] - software_version = software_info_list[1] - software_main_version = self.get_main_version(software_version) - if isapp: - software_type = SType.APP + return SoftwareType.APP else: - software_type = self.get_software_type(software_name, compiler_mpi_info) - software_info = { - "sname":software_name, - "sversion": software_version, - "mversion": software_main_version, - "type" : software_type, - "suffix": self.get_suffix(software_info_list) - } - if software_type == SType.LIB or software_type == SType.MPI or software_type == SType.APP: - software_info["is_use_mpi"] = self.is_contained_mpi(compiler_mpi_info) - software_info["use_compiler"] = self.get_used_compiler(compiler_mpi_info) - return software_info + return SoftwareType.LIB def get_compiler_info(self, compilers, compiler_mpi_info): compiler_info = {"cname":None, "cmversion": None, self.FULL_VERSION: None} @@ -250,7 +211,7 @@ done return False def add_mpi_path(self, software_info, install_path): - if not software_info['is_use_mpi']: + if not software_info.use_mpi: return install_path mpi_info = self.get_mpi_info() if mpi_info[self.FULL_VERSION] == None: @@ -265,28 +226,25 @@ done return install_path def get_install_path(self, software_info, env_info): - suffix = software_info['suffix'] - sversion = software_info['sversion'] - stype = software_info['type'] + sversion = software_info.full_version + stype = software_info.software_type cname = env_info['cname'] cfullver = env_info[self.FULL_VERSION] - if suffix != "": - software_info['sname'] += '-' + suffix - sname = software_info['sname'] - if stype == SType.MPI: + sname = software_info.name + if stype == SoftwareType.MPI: if self.IS_PRO: mpi_path = os.path.join(self.MPI_PATH, f"{sname}{sversion}-{cname}{cfullver}", sversion) if self.IS_NORMAL: mpi_path = os.path.join(self.MPI_PATH,sname, f"{sversion}-{cname}{cfullver}") print(f"mpi_path:{mpi_path}") return mpi_path - if stype == SType.COMPILER: + if stype == SoftwareType.COMPILER: install_path = os.path.join(self.COMPILER_PATH, f'{sname}/{sversion}') - elif stype == SType.UTIL: + elif stype == SoftwareType.UTIL: install_path = os.path.join(self.UTILS_PATH, f'{sname}/{sversion}') - elif stype == SType.MISC: + elif stype == SoftwareType.MISC: install_path = os.path.join(self.MISC_PATH, f'{sname}/{sversion}') - elif stype == SType.APP: + elif stype == SoftwareType.APP: install_path = os.path.join(self.APP_PATH, f'{sname}/{sversion}-{cname}{cfullver}') install_path = self.add_mpi_path(software_info, install_path) else: @@ -431,9 +389,9 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} return self.json.get(install_path) def gen_module_file(self, install_path, software_info, env_info): - sname = software_info['sname'] - sversion = software_info['sversion'] - stype = software_info['type'] + sname = software_info.name + sversion = software_info.full_version + stype = software_info.software_type cname = env_info['cname'] cfullversion = env_info[self.FULL_VERSION] module_file_content = self.get_module_file_content(install_path, sname, sversion) @@ -443,7 +401,7 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} if len(os.listdir(install_path)) == 0: print('module file did not generated because no file generated under install path') return '' - if stype == SType.MPI: + if stype == SoftwareType.MPI: compiler_str = cname + cfullversion software_str = sname + sversion if self.IS_PRO: @@ -455,7 +413,7 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} elif self.IS_NORMAL: module_path = os.path.join(self.MODULE_COMPILER_PATH, sname, f'{sversion}-{compiler_str}') else: - if stype == SType.COMPILER: + if stype == SoftwareType.COMPILER: software_str = sname + sversion if self.IS_PRO: module_path = os.path.join(self.MODULE_FILES, sname) @@ -465,29 +423,29 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} print(f'attach module file {attach_module_path} successfully generated.') elif self.IS_NORMAL: module_path = os.path.join(self.MODULE_COMPILER_PATH, sname, sversion) - elif stype == SType.UTIL: + elif stype == SoftwareType.UTIL: if self.IS_PRO: module_path = os.path.join(self.MODULE_FILES, sname) elif self.IS_NORMAL: - module_path = os.path.join(self.MODULE_TOOL_PATH, sname, sversion) - elif stype == SType.MISC: + module_path = os.path.join(self.MODULE_UTIL_PATH, sname, sversion) + elif stype == SoftwareType.MISC: if self.IS_PRO: module_path = os.path.join(self.MODULE_FILES, sname) elif self.IS_NORMAL: module_path = os.path.join(self.MODULE_MISC_PATH, sname, sversion) else: if self.IS_NORMAL: - if stype == SType.APP: + if stype == SoftwareType.APP: BASE_PATH = self.MODULE_APP_PATH else: BASE_PATH = self.MODULE_LIB_PATH elif self.IS_PRO: - if stype == SType.APP: + if stype == SoftwareType.APP: BASE_PATH = self.MODULE_APP_PATH else: BASE_PATH = self.MODULE_DEPS_PATH compiler_str = cname + cfullversion - if software_info['is_use_mpi']: + if software_info.use_mpi: mpi_info = self.get_mpi_info() mpi_str = mpi_info['name'] + mpi_info[self.FULL_VERSION] if self.IS_PRO: @@ -581,8 +539,8 @@ chmod +x {install_script} def add_install_info(self, software_info, install_path): software_dict = {} - software_dict['name'] = software_info['sname'] - software_dict['version'] = software_info['sversion'] + software_dict['name'] = software_info.name + software_dict['version'] = software_info.full_version software_dict['module_path'] = '' self.json.set(install_path, software_dict, True) @@ -601,22 +559,16 @@ chmod +x {install_script} compilers = {"GCC":self.gcc_detector.detect, "CLANG":self.clang_detector.detect, "NVC":self.nvcc_detector.detect, "ICC":self.icc_detector.detect, "BISHENG": self.clang_detector.detect} - software_path = self.remove_prefix(software_path) - # software_path should exists - if not isapp: - abs_software_path = self.check_software_path(software_path) - if not abs_software_path: return compiler_mpi_info = self.check_compiler_mpi(compilers.keys(), compiler_mpi_info) - if not compiler_mpi_info: return - software_info = self.get_software_info(software_path, compiler_mpi_info, isapp) - stype = software_info['type'] + software_info = self.software_factory.create_profile(software_path, compiler_mpi_info, isapp) + stype = software_info.software_type # get compiler name and version env_info = self.get_compiler_info(compilers, compiler_mpi_info) - if stype == SType.LIB or stype == SType.MPI or stype == SType.APP: + if stype == SoftwareType.LIB or stype == SoftwareType.MPI or stype == SoftwareType.APP: cmversion = env_info['cmversion'] cfullver = env_info[self.FULL_VERSION] if cmversion == None: - print(f"The specified {software_info['use_compiler']} Compiler not found!") + print(f"The specified {software_info.compiler_name} Compiler not found!") return False else: print(f"Use Compiler: {env_info['cname']} {cfullver}") @@ -634,7 +586,7 @@ chmod +x {install_script} "env_info":env_info } # get install script - self.install_package(abs_software_path, install_path, other_args) + self.install_package(software_info.install_script_path, install_path, other_args) # add install info self.add_install_info(software_info, install_path) # gen module file @@ -655,6 +607,7 @@ sh {depend_file} self.tool.prt_content("UNINSTALL " + software_info) remove_list = [] installed_dict = self.json.data + installed_dict.pop('MODE', None) for path, software_row in installed_dict.items(): if software_info in software_row['name']: remove_list.append((path, software_row)) diff --git a/src/installStrategy.py b/src/installStrategy.py index 344973ed7f3eb3f16a61aafed5f965eaeb54f983..5946eab737532ea1e67d5776ba89637e3b33e53a 100644 --- a/src/installStrategy.py +++ b/src/installStrategy.py @@ -33,6 +33,7 @@ class NormalPathStrategy(PathStrategy): 'MODULE_COMPILER_PATH': 'JARVIS_MODULES_COMPILER', 'MODULE_MISC_PATH': 'JARVIS_MODULES_MISC', 'MODULE_MPI_PATH': 'JARVIS_MODULES_MPI', + 'MODULE_UTIL_PATH': 'JARVIS_MODULES_UTIL', 'MODULE_MOD_PATH': 'JARVIS_MODULES_MODS', 'JOBSCRIPT_PATH': 'JARVIS_JOBSCRIPT' }) diff --git a/src/pathBuilder.py b/src/pathBuilder.py new file mode 100644 index 0000000000000000000000000000000000000000..e6b63a81af7363c312fb9e982484498cefb03fe4 --- /dev/null +++ b/src/pathBuilder.py @@ -0,0 +1,69 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from softwareTypes import SoftwareProfile, EnvironmentProfile +from collections import defaultdict + +class PathBuilder(ABC): + def __init__(self, base_config: 'DeploymentConfig'): + self.config = base_config + + @abstractmethod + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + pass + +class MpiPathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + folder_pattern = ( + self.config.pro_path_pattern if self.config.is_pro + else self.config.normal_path_pattern + ) + return self._construct_mpi_path(folder_pattern, software, env) + + def _construct_mpi_path(self, pattern: str, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + path_vars = { + 'name': software.name, + 'version': software.full_version, + 'compiler': env.compiler_name, + 'compiler_ver': env.compiler_version + } + return self.config.mpi_base / pattern.format(**path_vars) + +class CompilerPathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, _: EnvironmentProfile) -> Path: + return self.config.compiler_base / f"{software.name}/{software.version}" + +class LibPathBuilder(PathBuilder): + def build_path(self, software: SoftwareProfile, env: EnvironmentProfile) -> Path: + base_path = self.config.pro_lib_base if self.config.is_pro else self.config.normal_lib_base + path_segments = [ + f"{env.compiler_name}{env.compiler_version}", + *([f"mpi{env.mpi_version}"] if software.requires_mpi else []), + f"{software.name}/{software.version}" + ] + return base_path.joinpath(*path_segments) + +from collections import defaultdict + +class PathFactory: + _builder_registry = defaultdict(lambda: None) + + def __init__(self, config: 'DeploymentConfig'): + self.config = config + self._initialize_builders() + + def _initialize_builders(self): + self._builder_registry.update({ + SoftwareType.MPI: MpiPathBuilder(self.config), + SoftwareType.COMPILER: CompilerPathBuilder(self.config), + SoftwareType.LIB: LibPathBuilder(self.config), + SoftwareType.APP: AppPathBuilder(self.config), + SoftwareType.UTIL: UtilPathBuilder(self.config), + SoftwareType.MISC: MiscPathBuilder(self.config) + }) + + def get_builder(self, software_type: SoftwareType) -> PathBuilder: + + if builder := self._builder_registry[software_type]: + return builder + raise ValueError(f"No path builder registered for {software_type}") + diff --git a/src/softwareFactory.py b/src/softwareFactory.py new file mode 100644 index 0000000000000000000000000000000000000000..166a801beeebd41cd25dda5546a1f7f3292e109f --- /dev/null +++ b/src/softwareFactory.py @@ -0,0 +1,71 @@ +import os +from typing import Dict +from softwareTypeDetection import SoftwareTypeDetection,MPIStrategy,AppStrategy,DefaultStrategy +from softwareTypes import SoftwareProfile, SoftwareType + +class SoftwareFactory: + PACKAGE = 'package' + def __init__(self): + self.ROOT = os.getcwd() + self.PACKAGE_PATH = os.path.join(self.ROOT, self.PACKAGE) + self._strategies: Dict[str, SoftwareTypeDetection] = { + 'mpi': MPIStrategy(), + 'app': AppStrategy(), + 'default': DefaultStrategy() + } + + def create_profile(self, software_path: str, compiler_info: str, isapp=False) -> SoftwareProfile: + """创建软件信息对象的统一入口""" + (software_name, version, suffix) = self._parse_name_and_version(software_path) + software_type = self._determine_type(software_name, compiler_info, isapp) + profile = SoftwareProfile( + name=self._handle_suffix(software_name, suffix), + full_version=version, + major_version=self._parse_major_version(version), + software_type=software_type, + suffix=suffix, + use_mpi='+MPI' in compiler_info, + compiler_name=compiler_info.split('+')[0] if '+' in compiler_info else compiler_info, + install_script_path=self._get_install_script_path(software_name, version, software_type, suffix), + ) + return profile + + def _handle_suffix(self, name: str, suffix): + if suffix != "": + return f"{name}-{suffix}" + return name + + def _determine_type(self, name: str, compiler_info: str, isapp=False) -> SoftwareType: + for strategy in self._strategies.values(): + detected_type = strategy.detect(name, compiler_info, isapp) + if detected_type: + return detected_type + return SoftwareType.LIB + + @staticmethod + def _parse_major_version(version: str) -> str: + return version.split('.')[0] if version else '0' + + def _parse_name_and_version(self, software_path: str) -> str: + software_path = self._remove_prefix(software_path) + software_info = software_path.split('/') + suffix = "" + if len(software_info) >= 3: suffix = software_info[2] + return (software_info[0], software_info[1], suffix) + + @staticmethod + def _remove_prefix(software_path): + if software_path.startswith('package/') or software_path.startswith('./'): + software_path = software_path.replace('./', '', 1) + software_path = software_path.replace('package/', '', 1) + return software_path + + def _get_install_script_path(self, software_name, version, software_type, suffix): + if software_type == SoftwareType.APP: + return False + install_script_path = os.path.join(self.PACKAGE_PATH, software_name, version, suffix) + if not os.path.exists(install_script_path): + print(f"{software_path} not exist, are you sure the software lies in package dir?") + sys.exit() + return install_script_path + diff --git a/src/softwareTypeDetection.py b/src/softwareTypeDetection.py new file mode 100644 index 0000000000000000000000000000000000000000..6b92a889b3f078122dc467bd8f4a9d6e8ce0ab2c --- /dev/null +++ b/src/softwareTypeDetection.py @@ -0,0 +1,28 @@ +from abc import ABC, abstractmethod +from softwareTypes import SoftwareProfile, SoftwareType + +class SoftwareTypeDetection(ABC): + @abstractmethod + def detect(self, software_name: str, compiler_info: str, isapp = False) -> SoftwareType: + pass + +class MPIStrategy(SoftwareTypeDetection): + MPI_KEYWORDS = {'hmpi', 'openmpi', 'hpcx', 'mpich'} + + def detect(self, name: str, _: str, isapp = False) -> SoftwareType: + return SoftwareType.MPI if any(name.startswith(kw) for kw in self.MPI_KEYWORDS) else None + +class AppStrategy(SoftwareTypeDetection): + def detect(self, _: str, compiler_info: str, isapp = False) -> SoftwareType: + return SoftwareType.APP if isapp else None + +class DefaultStrategy(SoftwareTypeDetection): + TYPE_MAPPING = { + "COM": SoftwareType.COMPILER, + "ANY": SoftwareType.UTIL, + "MISC": SoftwareType.MISC + } + + def detect(self, _: str, compiler_info: str, isapp = False) -> SoftwareType: + return self.TYPE_MAPPING.get(compiler_info, SoftwareType.LIB) + diff --git a/src/softwareTypes.py b/src/softwareTypes.py index 8350271f32f2d8059b142db6c80e81d373e32d9e..bfee1931f36bb93c4e43ac5a6eadccd467467791 100644 --- a/src/softwareTypes.py +++ b/src/softwareTypes.py @@ -18,9 +18,10 @@ class SoftwareProfile: suffix: str = "" use_mpi: bool = False compiler_name: str = None + install_script_path: str = None @dataclass -class CompilerInfo: - name: str - full_version: str - major_version: str +class EnvironmentProfile: + compiler_name: str + compiler_version: str + compiler_major_version: str