diff --git a/data.config b/data.config index ea748cab3a6b9e48959fddc95f6ceec8a60d9bdd..7f98b048d3991279a435ccf565be4f13c3955d7b 100644 --- a/data.config +++ b/data.config @@ -7,33 +7,36 @@ qe/6.4 $JARVIS_PROXY/QEF/q-e/archive/refs/tags/qe-6.4.1.tar.gz q-e-qe-6.4.1.tar. [DEPENDENCY] set -e set -x -./jarvis -install kgcc/9.3.1 com module purge -module use ./software/modulefiles -module load kgcc/9.3.1 -export CC=`which gcc` -export CXX=`which g++` -export FC=`which gfortran` -./jarvis -install openmpi/4.1.2 gcc -module load openmpi/4.1.2 -#test if mpi is normal -./jarvis -bench mpi -tar -xzvf ${JARVIS_DOWNLOAD}/q-e-qe-6.4.1.tar.gz +module use $JARVIS_MODULES/modules +./jarvis -install hpckit/2025.3.30 any +module load hpckit/2025.3.30 +module load bisheng/compiler4.2.0/bishengmodule +module load bisheng/hmpi25.0.0/hmpi +[ ! -d q-e-qe-6.4.1 ] && tar -xzvf ${JARVIS_DOWNLOAD}/q-e-qe-6.4.1.tar.gz +exit 0 [ENV] module purge -module use ./software/modulefiles -module load kgcc/9.3.1 -module load openmpi/4.1.2 +module use $JARVIS_MODULES/modules +module load hpckit/2025.3.30 +module load bisheng/compiler4.2.0/bishengmodule +module load bisheng/hmpi25.0.0/hmpi +export QE_ROOT=${JARVIS_ROOT}/q-e-qe-6.4.1/ [APP] app_name = QE +app_version = 6.4.1 +compiler = bisheng+mpi build_dir = ${JARVIS_ROOT}/q-e-qe-6.4.1/ binary_dir = ${JARVIS_ROOT}/q-e-qe-6.4.1/bin/ -case_dir = ${JARVIS_ROOT}/workloads/QE/qe-test +case_dir = ${JARVIS_JOBSCRIPT}/QE/ [BUILD] -./configure F90=gfortran F77=gfortran MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp +set -e +set -x +#$PATCH_TOOL $JARVIS_TPL/top50/qe/6.4.1/opt.patch +./configure F90=flang F77=flang MPIF90=mpifort MPIF77=mpifort CC=mpicc FCFLAGS="-O3" CFLAGS="-O3" --with-scalapack=no --enable-openmp --prefix=$1 make -j pw make install @@ -45,7 +48,30 @@ run = mpirun --allow-run-as-root -x OMP_NUM_THREADS=1 -np 8 binary = pw.x -input test_3.in nodes = 1 -[PERF] -perf= -nsys= -ncu=--target-processes all \ No newline at end of file +[BATCH] +mpirun --allow-run-as-root -x OMP_NUM_THREADS=1 -np 8 $QE_ROOT/bin/pw.x -input test_3.in + +[JOB] +#!/bin/sh +#DSUB -n qe_test +#DSUB --job_type cosched:hmpi +#DSUB -A root.default +#DSUB -q root.default +#DSUB -N 1 +#DSUB -R cpu=128 +#DSUB -oo qe.%J.out +#DSUB -eo qe.%J.err + +##set runtime environment variables +module load QE +ulimit -s unlimited +ulimit -c unlimited +echo "----HOSTFILE generated---" +cat $CCS_HOST_FILE +echo "-------------------------" +EXEC_CMD="time -p mpirun $CCS_MPI_OPTIONS -n 32 -x OMP_NUM_THREADS=1 -mca pml ucx -mca btl ^vader,tcp,openib,uct -x UCX_TLS=self,sm,rc pw.x -input test_3.in" +echo "$EXEC_CMD" +date +$EXEC_CMD +date + diff --git a/init.sh b/init.sh index 0f421a8bae36c970a348aaffc38f9a74133eb3e0..ca490e5f651ce7663c8b305ef6910a207da6674f 100644 --- a/init.sh +++ b/init.sh @@ -1,26 +1,52 @@ #!/bin/bash CUR_PATH=$(pwd) chmod -R +x ./benchmark -chmod -R +x ./package +#chmod -R +x ./package chmod -R +x ./test chmod +x ./*.sh chmod +x jarvis mkdir -p tmp export JARVIS_ROOT=${CUR_PATH} -export JARVIS_COMPILER=${CUR_PATH}/software/compiler -export JARVIS_MPI=${CUR_PATH}/software/mpi -export JARVIS_LIBS=${CUR_PATH}/software/libs -export JARVIS_UTILS=${CUR_PATH}/software/utils -export JARVIS_DOWNLOAD=${CUR_PATH}/downloads -export JARVIS_MODULES=${CUR_PATH}/software/modulefiles -export JARVIS_MODULEDEPS=${CUR_PATH}/software/moduledeps +export JARVIS_SOFT_ROOT=$JARVIS_ROOT/share/software +export JARVIS_MODE=1 +if [ "$JARVIS_MODE" -eq 0 ]; then + #professional mode:Clustered Directory Structure + export JARVIS_COMPILER=${JARVIS_SOFT_ROOT}/compiler + export JARVIS_MPI=${JARVIS_SOFT_ROOT}/mpi + export JARVIS_LIBS=${JARVIS_SOFT_ROOT}/libs + export JARVIS_UTILS=${JARVIS_SOFT_ROOT}/utils + export JARVIS_MODULES=${JARVIS_SOFT_ROOT}/modulefiles + export JARVIS_MODULEDEPS=${JARVIS_SOFT_ROOT}/moduledeps + export JARVIS_MISC=${JARVIS_SOFT_ROOT}/misc + export JARVIS_APP=${JARVIS_SOFT_ROOT}/app + export JARVIS_MODULES_APP=${JARVIS_SOFT_ROOT}/modulefiles/app +elif [ "$JARVIS_MODE" -eq 1 ]; then + #normal mode: Flat Directory Structure + export JARVIS_COMPILER=${JARVIS_SOFT_ROOT}/soft/compiler + export JARVIS_MPI=${JARVIS_SOFT_ROOT}/soft/mpi + export JARVIS_LIBS=${JARVIS_SOFT_ROOT}/soft/lib + export JARVIS_UTILS=${JARVIS_SOFT_ROOT}/soft/tool + export JARVIS_MODULES=${JARVIS_SOFT_ROOT}/modulefile + export JARVIS_MODULEDEPS=${JARVIS_SOFT_ROOT}/moduledeps + export JARVIS_MISC=${JARVIS_SOFT_ROOT}/soft/misc + export JARVIS_APP=${JARVIS_SOFT_ROOT}/soft/app + export JARVIS_MODULES_LIB=${JARVIS_SOFT_ROOT}/modulefile/lib + export JARVIS_MODULES_TOOL=${JARVIS_SOFT_ROOT}/modulefile/tool + export JARVIS_MODULES_COMPILER=${JARVIS_SOFT_ROOT}/modulefile/compiler + export JARVIS_MODULES_MPI=${JARVIS_SOFT_ROOT}/modulefile/mpi + export JARVIS_MODULES_LIB=${JARVIS_SOFT_ROOT}/modulefile/lib + export JARVIS_MODULES_MISC=${JARVIS_SOFT_ROOT}/modulefile/misc + export JARVIS_MODULES_APP=${JARVIS_SOFT_ROOT}/modulefile/app + export JARVIS_MODULES_MODS=${JARVIS_SOFT_ROOT}/modulefile/modules + export JARVIS_JOBSCRIPT=${JARVIS_SOFT_ROOT}/jobscript +fi export JARVIS_TMP=/tmp +export JARVIS_DOWNLOAD=${CUR_PATH}/downloads export JARVIS_TMP_DOWNLOAD=${CUR_PATH}/tmp export JARVIS_EXE=${CUR_PATH}/exe +export JARVIS_TPL=${CUR_PATH}/template export JARVIS_PROXY=https://gh.ddlc.top/https://github.com - - export DOWNLOAD_TOOL=${CUR_PATH}/package/common/download.sh export CHECK_DEPS=${CUR_PATH}/package/common/check_deps.sh export CHECK_ROOT=${CUR_PATH}/package/common/check_root.sh diff --git a/package/hpckit/2025.3.30/install.sh b/package/hpckit/2025.3.30/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..9407e099848477f45a9abc8966e55a0c378aba41 --- /dev/null +++ b/package/hpckit/2025.3.30/install.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +export hpckit_ver="25.0.0" +../meta.sh $1 diff --git a/package/zlib/1.2.11/install.sh b/package/zlib/1.2.11/install.sh index 39c232ffcccb340aec5a1edf8e28f46a0a86dc1c..8387d3dcda6de44bbe4b6d5b142189c5c5ba3107 100755 --- a/package/zlib/1.2.11/install.sh +++ b/package/zlib/1.2.11/install.sh @@ -1,6 +1,7 @@ #!/bin/bash set -x set -e +export DEPENDENCIES="openblas" zlib_ver='1.2.11' . ${DOWNLOAD_TOOL} -u https://zlib.net/fossils/zlib-${zlib_ver}.tar.gz cd ${JARVIS_TMP} diff --git a/src/analysisService.py b/src/analysisService.py index 38cf0cdff18653c70a406ff2409bacc0a89f5b29..29be3cef1223981eda4451d61bb8ac5d64e7cf35 100644 --- a/src/analysisService.py +++ b/src/analysisService.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- - from machineService import MachineService from configService import ConfigService from downloadService import DownloadService @@ -16,11 +15,11 @@ from loopService import LoopService class AnalysisService: def __init__(self): + self.jenv = EnvService() self.jmachine = MachineService() self.jconfig = ConfigService() self.jdownload = DownloadService() self.jinstall = InstallService() - self.jenv = EnvService() self.jbuild = BuildService() self.jrun = RunService() self.jperf = PerfService() @@ -106,4 +105,4 @@ class AnalysisService: def gen_simucode(self): self.jloop.get_simulate_code() - \ No newline at end of file + diff --git a/src/buildService.py b/src/buildService.py index 4ec097ae8ec8b3761d7430e8765539fe3ff645cf..b9b9ec62e21ec6f7571d28f7d257e0852bdbade9 100644 --- a/src/buildService.py +++ b/src/buildService.py @@ -1,33 +1,82 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- from dataService import DataService from executeService import ExecuteService from toolService import ToolService +from commandBuilder import CommandBuilder +from installService import InstallService +import subprocess +import os class BuildService: + DEFAULT_APP_NAME = "unknown" + DEFAULT_APP_VERSION = "1.0.0" + DEFAULT_COMPILER = "gcc" + BUILD_FILE_NAME = 'build.sh' + CLEAN_FILE_NAME = 'clean.sh' + def __init__(self): - self.hpc_data = DataService() + self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() + self.command = CommandBuilder() + self.installService = InstallService() + + def get_build_file(self): + if self.ds.app_config.build_dir: + build_root_path = self.ds.app_config.build_dir + else: + build_root_path = self.ds.root_path + return os.path.join(build_root_path, self.BUILD_FILE_NAME) + + def get_clean_file(self): + if self.ds.app_config.build_dir: + clean_root_path = self.ds.app_config.build_dir + else: + clean_root_path = self.ds.root_path + return os.path.join(clean_root_path, self.CLEAN_FILE_NAME) + def clean(self): - print(f"start clean {DataService.app_name}") - clean_cmd=self.hpc_data.get_clean_cmd() - clean_file = 'clean.sh' - self.tool.write_file(clean_file, clean_cmd) - run_cmd = f''' -chmod +x {clean_file} -./{clean_file} -''' - self.exe.exec_raw(run_cmd) + clean_cmd = self.ds.get_clean_cmd() + self._execute_script(self.get_clean_file(), clean_cmd, "clean") + + def inject_env(self): + env_cmd = self.command.env_activation() + self.exe.exec_inject(env_cmd) def build(self): - print(f"start build {DataService.app_name}") - build_cmd = self.hpc_data.get_build_cmd() - build_file = 'build.sh' - self.tool.write_file(build_file, build_cmd) - run_cmd = f''' -chmod +x {build_file} -./{build_file} -''' + self.inject_env() + app_name = self.ds.get_app_name() or self.DEFAULT_APP_NAME + app_version = self.ds.get_app_version() or self.DEFAULT_APP_VERSION + app_compiler = self.ds.get_app_compiler() or self.DEFAULT_COMPILER + + result = self.installService.install([f"{app_name}/{app_version}", app_compiler], True) + self._handle_install_result(result) + + def _handle_install_result(self, result): + install_path = result["install_path"] + software_info = result["software_info"] + env_info = result["env_info"] + build_cmd = self.command.build() + self._execute_script(self.get_build_file(), build_cmd, "build", install_path) + self.installService.add_install_info(software_info, install_path) + self._update_dependencies() + self.installService.gen_module_file(install_path, software_info, env_info) + + def _update_dependencies(self): + loaded_modules = self.installService.get_loaded_modules() + self.installService.dependencies = " ".join(loaded_modules) + + def _execute_script(self, file_name, command, action, *args): + print(f"start {action} {self.ds.get_app_name()}") + self.tool.write_file(file_name, command) + run_cmd = f"chmod +x {file_name}\nsh {file_name} " + " ".join(args) self.exe.exec_raw(run_cmd) + +# Example usage: +if __name__ == "__main__": + build_service = BuildService() + build_service.clean() + build_service.build() + diff --git a/src/commandBuilder.py b/src/commandBuilder.py new file mode 100644 index 0000000000000000000000000000000000000000..b9ebf75a195214db6265b1e52046bd93b9d4b95d --- /dev/null +++ b/src/commandBuilder.py @@ -0,0 +1,108 @@ +import os +from dataService import DataService +from toolService import ToolService +from typing import List, Dict, Tuple + +class CommandBuilder: + """命令构造组件 (与DataService解耦)""" + def __init__(self): + self.ds = DataService() + self.tool_service = ToolService() + self.ROOT = os.getcwd() + + def env_generate(self) -> str: + """环境文件生成命令 (模板方法)""" + env_file = self.ds.get_env_file() + self.tool_service.write_file(env_file, self.ds.env_content) + print(f"ENV FILE {env_file} GENERATED.") + return f'chmod +x {env_file}' + + def env_activation(self) -> str: + """环境激活命令 (模板方法)""" + env_file = self.ds.get_env_file() + return f'source {env_file}' + + def clean(self) -> str: + """构建清理命令链""" + return self._chain_commands([ + self.env_activation, + f'cd {self.ds.app_config.build_dir}', + self.ds.clean_cmd + ]) + + def build(self) -> str: + """构建编译命令链""" + return self._chain_commands([ + f'cd {self.ds.app_config.build_dir}', + self.ds.build_cmd + ]) + + def run(self) -> str: + """生成运行命令核心逻辑""" + nodes = int(self.ds.run_cmd.get('nodes', 1)) + base_cmd = self.ds.run_cmd.get('run', '') + + # 动态添加hostfile参数 + hostfile = f'--hostfile {self.ds.root_path}/hostfile' if nodes > 1 else '' + cmd = base_cmd.replace('mpirun', f'mpirun {hostfile}') if 'mpi' in base_cmd else base_cmd + + # 拼接二进制路径和参数 + binary_path = os.path.join( + self.ds.app_config.binary_dir, + self.ds.binary_file[0] + ) + return f'{cmd} {binary_path} {self.ds.binary_file[1]}' + + def full_run(self) -> str: + """完整运行环境命令链""" + return self._chain_commands([ + self.env_activation(), + f'cd {self.ds.app_config.case_dir}', + self.run() + ]) + + def batch_run(self): + batch_file_path = self.ds.get_batch_run_file() + print(f"start batch run {self.ds.get_app_name()}") + batch_content = self._chain_commands([ + self.env_activation(), + f'cd {self.ds.app_config.case_dir}', + self.ds.get_batch_cmd() + ]) + self.tool_service.write_file(batch_file_path, batch_content) + return self._chain_commands([ + f"chmod +x {batch_file_path}", + f"sh {batch_file}" + ]) + + def job_run(self, num): + job_file_path = self.ds.get_job_run_file() + print(f"start job run {self.ds.get_app_name}") + job_cmd = self.ds.get_job_cmd() if num == 1 else self.ds.get_job2_cmd() + job_content = f''' +{self.env_activation()} +cd {self.ds.app_config.case_dir} +cat > job_run.sh << \EOF +{job_cmd} +EOF + +chmod +x job_run.sh +if type djob >/dev/null 2>&1;then + dsub -s job_run.sh +elif type sbatch >/dev/null 2>&1;then + sbatch job_run.sh +else + echo "dsub not exists." +fi +''' + self.tool_service.write_file(job_file_path, job_content) + return self._chain_commands([ + f"chmod +x {job_file_path}", + f"sh {job_file_path}" + ]) + + @staticmethod + def _chain_commands(commands: List[str]) -> str: + """命令链拼接工具 (自动过滤空行)""" + return '\n'.join([cmd for cmd in commands if cmd.strip()]) + diff --git a/src/configService.py b/src/configService.py index ad73f3a5361b6de0a6dce810cc45cac549a3d468..396a275b5ad4ceb5e1f9554e4ef767ddd9eb4bcc 100644 --- a/src/configService.py +++ b/src/configService.py @@ -2,15 +2,14 @@ # -*- coding: utf-8 -*- import os from dataService import DataService -from executeService import ExecuteService from toolService import ToolService class ConfigService: def __init__(self): - self.exe = ExecuteService() + self.ds = DataService() self.tool = ToolService() self.ROOT = os.getcwd() - self.meta_path = os.path.join(self.ROOT, DataService.meta_file) + self.meta_path = os.path.join(self.ROOT, self.ds.META_FILE) def switch_config(self, config_file): print(f"Switch config file to {config_file}") @@ -20,7 +19,7 @@ class ConfigService: return contents = self.tool.read_file(config_file) # keys should contains in config - keys = ["DOWNLOAD","DEPENDENCY","ENV","APP","BUILD","RUN"] + keys = self.ds.KEY_CONFIG_SECTIONS for key in keys: if f"[{key}]" not in contents: print(f"key [{key}] not found in {config_file}, switch failed.") diff --git a/src/dataService.py b/src/dataService.py index 09e888db1e7c5b354af37fc49def68d07040f514..50128e24781ca6b245bd5f78c0fd467c569450d0 100644 --- a/src/dataService.py +++ b/src/dataService.py @@ -4,215 +4,282 @@ import os import platform from toolService import ToolService +from typing import List, Dict, Tuple class Singleton(type): + """单例元类 (线程不安全基础版)""" + _instances = {} - def __init__(self, name, bases, dictItem): - super(Singleton,self).__init__(name,bases, dictItem) - self._instance = None - - def __call__(self, *args, **kwargs): - if self._instance is None: - self._instance = super(Singleton,self).__call__(*args, **kwargs) - return self._instance - -class DataService(object,metaclass=Singleton): - # Hardware Info - avail_ips='' - # Dependent Software environment Info - dependency = '' - module_content='' - env_file = 'env.sh' - - # Application Info - app_name = '' - build_dir = '' - binary_dir = '' - binary_file = '' - case_dir = '' - - # cmd info - build_cmd = '' - clean_cmd = '' - run_cmd = {} - batch_cmd = '' - loop_cmd = '' - job_cmd = '' - job2_cmd = '' - #Other Info - env_config_file = 'JARVIS_CONFIG' - config_file = 'data.config' - meta_file = '.meta' - root_path = os.getcwd() - download_info = '' - #perf info - kperf_para = '' - perf_para = '' - nsys_para = '' - ncu_para = '' - hpccollect_para = '' - hpcreport_para = '' - def get_abspath(self, relpath): - return os.path.join(DataService.root_path, relpath) + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] +class AppConfig: + """应用配置数据类 (Immutable)""" + __slots__ = ['name', 'version', 'compiler', 'build_dir', 'binary_dir', 'case_dir'] + + def __init__(self, + name: str = "", + version: str = "", + compiler: str = "", + build_dir: str = "", + binary_dir: str = "", + case_dir: str = ""): + self.name = name + self.version = version + self.compiler = compiler + self.build_dir = os.path.expandvars(build_dir) + self.binary_dir = os.path.expandvars(binary_dir) + self.case_dir = os.path.expandvars(case_dir) + +class PerfConfig: + """性能工具参数配置""" def __init__(self): - self.isARM = platform.machine() == 'aarch64' - self.tool = ToolService() - self.data_process() + self.kperf: str = "" + self.perf: str = "" + self.nsys: str = "" + self.ncu: str = "" + self.hpccollect: str = "" + self.hpcreport: str = "" + +class DataService(metaclass=Singleton): + + """配置中心服务 (单例模式)""" + + # 常量定义 + ENV_CONFIG_VAR = 'JARVIS_CONFIG' + DEFAULT_CONFIG_FILE = 'data.config' + META_FILE = '.meta' + ENV_FILE = 'env.sh' + BATCH_FILE = 'batch_run.sh' + DEPEND_FILE = 'depend_install.sh' + JOB_FILE = 'job_generate.sh' + CONFIG_SECTIONS = [ + '[SERVER]', '[DOWNLOAD]', '[DEPENDENCY]', '[ENV]', + '[APP]', '[BUILD]', '[CLEAN]', '[RUN]', + '[BATCH]', '[LOOP]', '[JOB]', '[JOB2]', '[PERF]' + ] + KEY_CONFIG_SECTIONS = [ + "DOWNLOAD","DEPENDENCY","ENV","APP","BUILD","RUN" + ] + + def __init__(self): + # 基础设施信息 + self.available_ips: str = "" + self.dependency: str = "" + self.env_content: str = "" + # 应用配置 + self.app_config = AppConfig() + self.build_dir: str = "" + self.binary_dir: str = "" + self.case_dir: str = "" + # 命令配置 + self.build_cmd: str = "" + self.clean_cmd: str = "" + self.run_cmd: Dict[str, str] = {} + self.batch_cmd: str = "" + self.loop_cmd: str = "" + self.job_cmd: str = "" + self.job2_cmd: str = "" + # 性能工具 + self.perf_config = PerfConfig() + + # 系统状态 + self.is_arm: bool = platform.machine() == 'aarch64' + self.root_path: str = os.getcwd() + self.tool_service: ToolService = ToolService() + # 初始化流程 + self._load_configurations() + # 优先读取环境变量的JARVIS_CONFIG配置,方便多人同时操控 # 然后读取.meta文件中的值 # 最后读取data.config中的值 - def get_config_file_name(self): - CONFIG_ENV = os.getenv(DataService.env_config_file) - if CONFIG_ENV is not None: - print("LOAD Config file from ENV:", CONFIG_ENV) - return CONFIG_ENV - if not os.path.exists(DataService.meta_file): - return DataService.config_file - return self.tool.read_file(DataService.meta_file) - - def get_data_config(self): - file_name = self.get_config_file_name() - file_path = self.get_abspath(file_name) - if not os.path.exists(file_path): - print("config file not found, switch to default data.config.") - file_path = self.get_abspath(DataService.config_file) - with open(file_path, encoding='utf-8') as file_obj: - contents = file_obj.read() - return contents.strip() - - def is_empty(self, content): - return len(content) == 0 or content.isspace() or content == '\n' - - def read_rows(self, rows, start_row, needs_strip=True): - data = '' - row = rows[start_row] - if needs_strip: - row = row.strip() - while not row.startswith('['): - if not self.is_empty(row): - data += row + '\n' - start_row += 1 - if start_row == len(rows): - break - row = rows[start_row] - if needs_strip: - row = row.strip() - return start_row, data - - def read_rows_kv(self, rows, start_row): - data = {} - row = rows[start_row].strip() - while not row.startswith('['): - if '=' in row: - key, value = row.split('=', 1) - data[key.strip()] = value.strip() - start_row += 1 - if start_row == len(rows): - break - row = rows[start_row].strip() - return start_row, data - - def set_app_info(self, data): - DataService.app_name = data['app_name'] - DataService.build_dir = data['build_dir'] - DataService.binary_dir = data['binary_dir'] - DataService.case_dir = data['case_dir'] + def _get_config_path(self) -> str: + """确定配置文件优先级""" + env_config = os.getenv(self.ENV_CONFIG_VAR) + if env_config: + print(f"LOAD Config from ENV: {env_config}") + return self._resolve_path(env_config) + + if os.path.exists(self.META_FILE): + meta_config = self.tool_service.read_file(self.META_FILE) + return self._resolve_path(meta_config) + + return self._resolve_path(self.DEFAULT_CONFIG_FILE) + + def _resolve_path(self, rel_path: str) -> str: + """处理路径解析""" + abs_path = os.path.join(self.root_path, rel_path) + if not os.path.exists(abs_path): + print(f"Config file {abs_path} not found, fallback to default.") + abs_path = os.path.join(self.root_path, self.DEFAULT_CONFIG_FILE) + return abs_path + + def _load_configurations(self): + """配置加载总入口""" + config_path = self._get_config_path() + with open(config_path, 'r', encoding='utf-8') as f: + raw_data = self._parse_config_file(f.readlines()) + self._map_config_data(raw_data) + + def _parse_config_file(self, lines: List[str]) -> Dict[str, str]: + """解析配置文件结构""" + config = {} + current_section = None + buffer = [] + + for line in lines: + line = line.strip() + if line in self.CONFIG_SECTIONS: + if current_section: + config[current_section] = '\n'.join(buffer).strip() + current_section = line + buffer = [] + elif current_section: + buffer.append(line) + + if current_section and buffer: + config[current_section] = '\n'.join(buffer).strip() + + return config + + def _map_config_data(self, config_data: Dict[str, str]): + """映射配置数据到对象属性""" + # 基础信息 + self.available_ips = config_data.get('[SERVER]', '') + self.dependency = config_data.get('[DEPENDENCY]', '') + self.env_content = config_data.get('[ENV]', '') + self.download_info = config_data.get('[DOWNLOAD]', '') + + # 命令配置 + self._parse_command_sections(config_data) + + # 应用配置 + app_data = self._parse_app_section(config_data.get('[APP]', '')) + + # 性能配置 + perf_data = self._parse_perf_section(config_data.get('[PERF]', '')) + # 处理二进制命令 + self._process_binary_command() + + def _parse_command_sections(self, config_data: Dict[str, str]): + """解析所有命令类型段落""" + self.build_cmd = self._parse_section_content(config_data.get('[BUILD]', '')) + self.clean_cmd = self._parse_section_content(config_data.get('[CLEAN]', '')) + self.run_cmd = self._parse_kv_section(config_data.get('[RUN]', '')) + self.batch_cmd = self._parse_section_content(config_data.get('[BATCH]', '')) + self.loop_cmd = self._parse_section_content(config_data.get('[LOOP]', '')) + self.job_cmd = self._parse_section_content(config_data.get('[JOB]', '')) + self.job2_cmd = self._parse_section_content(config_data.get('[JOB2]', '')) + + def _parse_app_section(self, app_content: str): + """解析应用配置段落""" + app_data = self._parse_kv_section(app_content) + self.app_config = AppConfig( + name=app_data.get('app_name', ''), + version=app_data.get('app_version', ''), + compiler=app_data.get('compiler', ''), + build_dir=app_data.get('build_dir', ''), + binary_dir=app_data.get('binary_dir', ''), + case_dir=app_data.get('case_dir', '') + ) + + def _parse_perf_section(self, perf_content: str): + """解析性能配置段落""" + perf_data = self._parse_kv_section(perf_content) + self.perf_config.kperf = perf_data.get('kperf', '') + self.perf_config.perf = perf_data.get('perf', '') + self.perf_config.nsys = perf_data.get('nsys', '') + self.perf_config.ncu = perf_data.get('ncu', '') + + def _process_binary_command(self): + """处理二进制命令分解""" + if 'binary' in self.run_cmd: + parts = self.run_cmd['binary'].split(' ', 1) + self.binary_file = parts[0] + self.binary_para = parts[1] if len(parts) > 1 else '' + + @staticmethod + def _parse_kv_section(content: str) -> Dict[str, str]: + """解析键值对格式的配置段落""" + kv_dict = {} + for line in content.split('\n'): + if '=' in line: + kv = line.split('=', 1) + kv_dict[kv[0].strip()] = kv[1].strip() + return kv_dict + + @staticmethod + def _parse_section_content(content: str) -> str: + """解析多行文本段落""" + return '\n'.join(line.strip() for line in content.split('\n') if line.strip()) + + @staticmethod + def _is_empty(content: str) -> bool: + """判断字符串是否为空内容""" + return not content.strip() + + def get_app_name(self): + return self.app_config.name + + def get_app_version(self): + return self.app_config.version + + def get_app_compiler(self): + return self.app_config.compiler - def set_perf_info(self, data): - DataService.kperf_para = data['kperf'] if 'kperf' in data else '' - DataService.perf_para = data['perf'] if 'perf' in data else '' - DataService.nsys_para = data['nsys'] if 'nsys' in data else '' - DataService.ncu_para = data['ncu'] if 'ncu' in data else '' - DataService.hpccollect_para = data['hpccollect'] if 'hpccollect' in data else '' - DataService.hpcreport_para = data['hpcreport'] if 'hpcreport' in data else '' - - def split_two_part(self, data): - split_list = data.split(' ', 1) - first = split_list[0] - second = '' - if len(split_list) > 1: - second = split_list[1] - return (first, second) - - def data_integration(self, config_data): - DataService.avail_ips = config_data.get('[SERVER]','') - DataService.download_info = config_data.get('[DOWNLOAD]','') - DataService.dependency = config_data.get('[DEPENDENCY]','') - DataService.module_content = config_data.get('[ENV]','') - DataService.build_cmd = config_data.get('[BUILD]','') - DataService.clean_cmd = config_data.get('[CLEAN]','') - DataService.run_cmd = config_data.get('[RUN]','') - DataService.batch_cmd = config_data.get('[BATCH]','') - DataService.loop_cmd = config_data.get('[LOOP]','') - DataService.job_cmd = config_data.get('[JOB]','') - DataService.job2_cmd = config_data.get('[JOB2]','') - data = config_data.get('[APP]','') - perf_data = config_data.get('[PERF]','') - self.set_app_info(data) - self.set_perf_info(perf_data) - DataService.binary_file, DataService.binary_para = self.split_two_part(DataService.run_cmd['binary']) - - def data_process(self): - contents = self.get_data_config() - rows = contents.split('\n') - rowIndex = 0 - handlers = { - '[SERVER]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1), - '[DOWNLOAD]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1), - '[DEPENDENCY]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1), - '[ENV]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1), - '[APP]': lambda rows, rowIndex: self.read_rows_kv(rows, rowIndex+1), - '[BUILD]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1, False), - '[CLEAN]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1), - '[RUN]': lambda rows, rowIndex: self.read_rows_kv(rows, rowIndex+1), - '[BATCH]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1), - '[LOOP]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1, False), - '[JOB]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1, False), - '[JOB2]': lambda rows, rowIndex: self.read_rows(rows, rowIndex+1, False), - '[PERF]': lambda rows, rowIndex: self.read_rows_kv(rows, rowIndex+1) - } - config_data = {} - while rowIndex < len(rows): - row = rows[rowIndex].strip() - if row in handlers.keys(): - rowIndex, config_data[row] = handlers[row](rows, rowIndex) - else: - rowIndex += 1 - self.data_integration(config_data) + def get_download_info(self): + return self.download_info + + def get_available_ips(self): + return self.available_ips + + def get_run_cmd(self, key): + return self.run_cmd.get(key) + + def get_batch_cmd(self): + return self.batch_cmd + + def get_job_cmd(self): + return self.job_cmd + + def get_job2_cmd(self): + return self.job2_cmd def get_clean_cmd(self): - return f''' -{self.get_env()} -cd {DataService.build_dir} -{DataService.clean_cmd} -''' - def get_env(self): - return f'''source ./init.sh -./jarvis -e -source ./{DataService.env_file}''' - - def get_build_cmd(self): - return f''' -{self.get_env()} -cd {DataService.build_dir} -{DataService.build_cmd} -''' - - def get_run(self): - nodes = int(DataService.run_cmd['nodes']) - run_cmd = DataService.run_cmd['run'] - hostfile = '' - if nodes > 1: - hostfile = f'--hostfile {DataService.root_path}/hostfile' - if 'mpi' in run_cmd: - run_cmd = run_cmd.replace('mpirun', f'mpirun {hostfile}') - binary = os.path.join(DataService.binary_dir, DataService.binary_file) - return f'''{run_cmd} {binary} {DataService.binary_para}''' - - def get_run_cmd(self): - return f''' -{self.get_env()} -cd {DataService.case_dir} -{self.get_run()} -''' + return self.clean_cmd + + def get_env_file(self): + if self.app_config.build_dir: + env_root_path = self.app_config.build_dir + else: + env_root_path = self.root_path + return os.path.join(env_root_path, self.ENV_FILE) + + def get_depend_file(self): + if self.app_config.build_dir: + depend_root_path = self.app_config.build_dir + else: + depend_root_path = self.root_path + return os.path.join(depend_root_path, self.DEPEND_FILE) + + def get_batch_run_file(self): + if self.app_config.case_dir: + batch_root_path = self.app_config.case_dir + else: + batch_root_path = self.root_path + return os.path.join(batch_root_path, self.BATCH_FILE) + + def get_job_run_file(self): + if self.app_config.case_dir: + job_root_path = self.app_config.case_dir + else: + job_root_path = self.root_path + return os.path.join(job_root_path, self.JOB_FILE) + + def get_dependency(self): + return self.dependency + + def get_binary_file(self): + return self.binary_file diff --git a/src/detectorService.py b/src/detectorService.py new file mode 100644 index 0000000000000000000000000000000000000000..4b99dfe90ecfb9dfa79fc70c59036d0ddcb5c1b2 --- /dev/null +++ b/src/detectorService.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +from abc import ABC, abstractmethod +from versionParser import VersionParser +from executeService import ExecuteService +from typing import Dict + +class DetectorService(ABC): + def __init__(self): + self.exe = ExecuteService() + + @abstractmethod + def detect(self) -> Dict: + pass + +class GCCDetector(DetectorService): + def detect(self) -> Dict: + gcc_info_list = self.exe.exec_get_output("gcc -v") + gcc_info = gcc_info_list[-1].strip() + version = VersionParser.parse(gcc_info) + if not version: + print("GCC not found, please install gcc first") + sys.exit() + name = 'gcc' + if 'kunpeng' in gcc_info.lower(): + name = 'kgcc' + profile = VersionParser.gen_compiler_profile(name, version) + return profile + +class ClangDetector(DetectorService): + def detect(self) -> Dict: + clang_info_list = self.exe.exec_get_output('clang -v') + clang_info = clang_info_list[0].strip() + version = VersionParser.parse(clang_info) + if not version: + print("clang not found, please install clang first") + sys.exit() + name = 'clang' + if 'bisheng' in clang_info.lower(): + name = 'bisheng' + profile = VersionParser.gen_compiler_profile(name, version) + return profile + +class NVCCDetector(DetectorService): + def detect(self) -> Dict: + nv_info_list = self.exe.exec_get_output("nvcc -v") + nv_info = nv_info_list[-1].strip() + version = VersionParser.parse(nv_info) + if not version: + print("nvcc not found, please install cuda first") + sys.exit() + name = 'nvcc' + profile = VersionParser.gen_compiler_profile(name, version) + return profile + +class ICCDetector(DetectorService): + def detect(self) -> Dict: + icc_info_list = self.exe.exec_get_output("icc --version") + icc_info = icc_info_list[-1].strip() + version = VersionParser.parse(icc_info) + if not version: + print("ICC not found, please install icc first") + sys.exit() + name = 'icc' + profile = VersionParser.gen_compiler_profile(name, version) + return profile + +class HMPIDetector(DetectorService): + def get_hmpi_version(self, hmpi_v3_info): + if hmpi_v3_info != "": + ucg_path = self.exe.exec_get_output('which ucg_info')[0] + else: + ucg_path = self.exe.exec_get_output('which ucx_info')[0] + ver_dict = {('2','2.0.0'): ('1','1.3.0')} + ucg_path = os.path.dirname(ucg_path) + ucg_path = os.path.dirname(ucg_path) + libucg_path = os.path.join(ucg_path, "lib") + libucg_so_flag = "libucg.so." + version = None + for file_name in os.listdir(libucg_path): + if libucg_so_flag in file_name: + version = VersionParser.parse(file_name) + if version in ver_dict: + return ver_dict[version] + elif version: + break + return version + + def detect(self) -> Dict: + hmpi_v2_info = (self.exe.exec_get_output('(ucx_info -c | grep -i BUILT)')[0]).upper() + hmpi_v3_info = (self.exe.exec_get_output('(ucg_info -c | grep -i PLANC)')[0]).upper() + if "BUILT" not in hmpi_v2_info and "PLANC" not in hmpi_v3_info: + return None + name = 'hmpi' + version = self.get_hmpi_version(hmpi_v3_info) + profile = VersionParser.gen_mpi_profile(name, version) + return profile + +class OpenMPIDetector(DetectorService): + def detect(self) -> Dict: + mpi_info_list = self.exe.exec_get_output('mpirun -version') + mpi_info = mpi_info_list[0].strip() + name = 'openmpi' + version = VersionParser.parse(mpi_info) + if not version: + return None + profile = VersionParser.gen_mpi_profile(name, version) + return profile + +class MPICHDetector(DetectorService): + def detect(self) -> Dict: + mpi_info_list = self.exe.exec_get_output('mpirun -version') + mpi_info = "".join(mpi_info_list).strip() + name = 'mpich' + if name not in mpi_info: + return None + version = VersionParser.parse(mpi_info) + if not version: + return None + profile = VersionParser.gen_mpi_profile(name, version) + return profile + diff --git a/src/downloadService.py b/src/downloadService.py index 4d44bfc413777c99323f3ba1ab4e52627ff1256e..a00c5a4dd95501f73c77e683a2f4ff7ebf5d6687 100644 --- a/src/downloadService.py +++ b/src/downloadService.py @@ -7,11 +7,11 @@ from toolService import ToolService class DownloadService: def __init__(self): - self.hpc_data = DataService() + self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() self.ROOT = os.getcwd() - self.download_list = self.tool.gen_list(DataService.download_info) + self.download_list = self.tool.gen_list(self.ds.get_download_info()) self.download_path = os.path.join(self.ROOT, 'downloads') self.package_path = os.path.join(self.ROOT, 'package') diff --git a/src/envService.py b/src/envService.py index fbdfde030de7e766a1ebfeab79b64cc4cba6a59b..0bcc5a2158a2291ea6512c3bced6675b8ca90d04 100644 --- a/src/envService.py +++ b/src/envService.py @@ -2,20 +2,28 @@ # -*- coding: utf-8 -*- import os -from dataService import DataService -from toolService import ToolService +from commandBuilder import CommandBuilder from executeService import ExecuteService +from dataService import DataService + +class Singleton(type): + def __init__(self, name, bases, dictItem): + super(Singleton,self).__init__(name,bases, dictItem) + self._instance = None + + def __call__(self, *args, **kwargs): + if self._instance is None: + self._instance = super(Singleton,self).__call__(*args, **kwargs) + return self._instance -class EnvService: +class EnvService(object,metaclass=Singleton): def __init__(self): - self.hpc_data = DataService() - self.tool = ToolService() - self.ROOT = os.getcwd() + self.command = CommandBuilder() # 注入命令生成组件 self.exe = ExecuteService() + self.ds = DataService() + self.env() def env(self): - print(f"set environment {DataService.app_name}") - env_file = os.path.join(self.ROOT, DataService.env_file) - self.tool.write_file(env_file, DataService.module_content) - print(f"ENV FILE {DataService.env_file} GENERATED.") - self.exe.exec_raw(f'chmod +x {DataService.env_file}') + print(f"set environment...{self.ds.get_app_name()}") + env_cmd = self.command.env_generate() + self.exe.exec_raw(env_cmd) diff --git a/src/executeService.py b/src/executeService.py index a514e2c3330c74228d11858f0bee2be28f165362..a8532ab5fa31b260a5c7d829d595f8abee53fb44 100644 --- a/src/executeService.py +++ b/src/executeService.py @@ -5,6 +5,7 @@ import logging from asyncio.log import logger from datetime import datetime from toolService import ToolService +import subprocess LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s" DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p" @@ -17,6 +18,7 @@ class ExecuteService: self.tool = ToolService() self.flags = '*' * 80 self.end_flag = 'END: ' + self.ROOT = os.getcwd() # tools function def join_cmd(self, arrs): @@ -33,7 +35,7 @@ class ExecuteService: def exec_popen(self, cmd, isPrint=True): if isPrint: self.print_cmd(cmd) - output = os.popen(f"bash -c '{cmd}'").readlines() + output = os.popen(f"bash -c '{cmd} 2>&1'").readlines() return output def get_duration(self): @@ -59,4 +61,30 @@ class ExecuteService: return True def exec_raw(self, rows): - return self.exec_list(self.tool.gen_list(rows)) \ No newline at end of file + return self.exec_list(self.tool.gen_list(rows)) + + def exec_get_output(self, cmd): + tmp_path = os.path.join(self.ROOT, 'tmp') + tmp_file = os.path.join(tmp_path, 'tmp.txt') + self.tool.mkdirs(tmp_path) + cmd += f' &> {tmp_file}' + self.exec_popen(cmd, False) + info_list = self.tool.read_file(tmp_file).split('\n') + return info_list + + def exec_inject(self, cmd_str): + cmd = f'{cmd_str} && env -0' + result = subprocess.run( + cmd, + shell=True, + stdout=subprocess.PIPE, + text=True + ) + # 处理错误并注入变量 + if result.returncode != 0: + print(f"execute {cmd} failed: {result.stderr}") + return + for line in result.stdout.strip('\0').split('\0'): + if '=' in line: + key, value = line.split('=', 1) + os.environ[key] = value diff --git a/src/initService.py b/src/initService.py new file mode 100644 index 0000000000000000000000000000000000000000..e3ce35415d31b00a4129f66abd9b572c9aac1bf4 --- /dev/null +++ b/src/initService.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os + +from executeService import ExecuteService + +class Singleton(type): + def __init__(self, name, bases, dictItem): + super(Singleton,self).__init__(name,bases, dictItem) + self._instance = None + + def __call__(self, *args, **kwargs): + if self._instance is None: + self._instance = super(Singleton,self).__call__(*args, **kwargs) + return self._instance + +class InitService(object,metaclass=Singleton): + INIT_FILE = 'init.sh' + def __init__(self): + self.exe = ExecuteService() + self.init_jarvis() + + def jarvis_activation(self) -> str: + """环境激活命令 (模板方法)""" + return f'source ./{self.INIT_FILE}' + + def init_jarvis(self): + init_cmd = self.jarvis_activation() + self.exe.exec_inject(init_cmd) + print("JARVIS IS READY ^_^") + + + diff --git a/src/installService.py b/src/installService.py index 778ff83f1681a5b1d5a593b9c3d8b14c761c12e2..efc81cce88c03c8306679405d160240dd81e9f30 100644 --- a/src/installService.py +++ b/src/installService.py @@ -6,151 +6,161 @@ import re import fnmatch from enum import Enum from glob import glob +from typing import Tuple, Optional, Dict from dataService import DataService from toolService import ToolService from executeService import ExecuteService from jsonService import JSONService +from installStrategy import PathStrategyFactory +from installTypes import InstallMode +from versionParser import VersionParser +from detectorService import GCCDetector, ClangDetector,NVCCDetector,ICCDetector +from detectorService import HMPIDetector, OpenMPIDetector, MPICHDetector + +class Singleton(type): + + def __init__(self, name, bases, dictItem): + super(Singleton,self).__init__(name,bases, dictItem) + self._instance = None + + def __call__(self, *args, **kwargs): + if self._instance is None: + self._instance = super(Singleton,self).__call__(*args, **kwargs) + return self._instance + class SType(Enum): COMPILER = 1 MPI = 2 UTIL = 3 LIB = 4 + MISC = 5 + APP = 6 -class InstallService: +class InstallService(object,metaclass=Singleton): + PACKAGE = 'package' + FULL_VERSION='full_version' def __init__(self): - self.hpc_data = DataService() + self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() self.ROOT = os.getcwd() - self.PACKAGE = 'package' - self.FULL_VERSION='fullver' - self.PACKAGE_PATH = os.path.join(self.ROOT, self.PACKAGE) - self.SOFTWARE_PATH = os.path.join(self.ROOT, 'software') + + self.gcc_detector = GCCDetector() + self.clang_detector = ClangDetector() + self.icc_detector = ICCDetector() + self.nvcc_detector = NVCCDetector() + + self.hmpi_detector = HMPIDetector() + self.openmpi_detector = OpenMPIDetector() + self.mpich_detector = MPICHDetector() + + self._mode = self._detect_mode() + self._paths = self._init_paths() self.INSTALL_INFO_PATH = os.path.join(self.SOFTWARE_PATH, "install.json") - self.COMPILER_PATH = os.path.join(self.SOFTWARE_PATH, 'compiler') - self.LIBS_PATH = os.path.join(self.SOFTWARE_PATH, 'libs') - self.MODULE_DEPS_PATH = os.path.join(self.SOFTWARE_PATH, 'moduledeps') - self.MODULE_FILES = os.path.join(self.SOFTWARE_PATH, 'modulefiles') - self.MPI_PATH = os.path.join(self.SOFTWARE_PATH, 'mpi') - self.UTILS_PATH = os.path.join(self.SOFTWARE_PATH, 'utils') - self.json = JSONService(self.INSTALL_INFO_PATH) - - def get_version_info(self, info, reg = r'(\d+)\.(\d+)\.(\d+)'): - matched_group = re.search(reg ,info) - if not matched_group: - return None - mversion = matched_group.group(1) - mid_ver = matched_group.group(2) - last_ver = matched_group.group(3) - return ( mversion, f'{mversion}.{mid_ver}.{last_ver}') - - def gen_compiler_dict(self, cname, version): - return {"cname": cname, "cmversion": version[0], self.FULL_VERSION: version[1]} - - def gen_mpi_dict(self, name, version): - return {"name": name, "mversion": version[0], self.FULL_VERSION: version[1]} - - # some command don't generate output, must redirect to a tmp file - def get_cmd_output(self, cmd): - tmp_path = os.path.join(self.ROOT, 'tmp') - tmp_file = os.path.join(tmp_path, 'tmp.txt') - self.tool.mkdirs(tmp_path) - cmd += f' &> {tmp_file}' - self.exe.exec_popen(cmd, False) - info_list = self.tool.read_file(tmp_file).split('\n') - return info_list - - def get_gcc_info(self): - gcc_info_list = self.get_cmd_output('gcc -v') - gcc_info = gcc_info_list[-1].strip() - version = self.get_version_info(gcc_info) - if not version: - print("GCC not found, please install gcc first") + self._is_first_install = self._set_first_install() + self.IS_PRO = self._mode == InstallMode.PRO + self.IS_NORMAL = self._mode == InstallMode.NORMAL + self.PACKAGE_PATH = os.path.join(self.ROOT, self.PACKAGE) + self.dependencies = False + self.LINK_FILE = os.path.join(self.MODULE_FILES,'linkpathtomodules.sh') + self._create_install_json(self.INSTALL_INFO_PATH) + self._validate_installation() + self._prepare_infrastructure() + + def _create_install_json(self, filename): + if self._is_first_install: + self.tool.write_file(filename, "{}") + self.json = JSONService(filename) + self.json.set("MODE", self._mode, True) + return + # 处理老的目录结构 + self.json = JSONService(filename) + stored_mode = self.json.get('MODE') + if not stored_mode: + self.json.set("MODE", InstallMode.PRO.value, True) + + def _set_first_install(self): + if not os.path.exists(self.INSTALL_INFO_PATH): + return True + return False + + def _detect_mode(self) -> InstallMode: + """安全模式检测""" + env_mode = os.getenv('JARVIS_MODE') + if env_mode not in (InstallMode.PRO.value, InstallMode.NORMAL.value): + print(f"Invalid JARVIS_MODE: {env_mode}") sys.exit() - name = 'gcc' - if 'kunpeng' in gcc_info.lower(): - name = 'kgcc' - return self.gen_compiler_dict(name, version) - - def get_clang_info(self): - clang_info_list = self.get_cmd_output('clang -v') - clang_info = clang_info_list[0].strip() - version = self.get_version_info(clang_info) - if not version: - print("clang not found, please install clang first") + modes = {"1": "normal", "0": "professional"} + print(f"current MODE: {modes.get(env_mode)}") + return InstallMode(env_mode) + + def _init_paths(self) -> Dict[str, str]: + """路径初始化策略化""" + strategy = PathStrategyFactory.create(self._mode) + self._paths = strategy.get_paths() + for attr, cur_path in self._paths.items(): + setattr(self, attr, cur_path) + + def _validate_installation(self) -> None: + """安装一致性校验""" + stored_mode = self.json.get('MODE') + if stored_mode and stored_mode != self._mode.value: + print(f"Mode conflict: Current {self._mode.value} vs Stored {stored_mode}") sys.exit() - name = 'clang' - if 'bisheng' in clang_info.lower(): - name = 'bisheng' - return self.gen_compiler_dict(name, version) - - def get_nvc_info(self): - return self.gen_compiler_dict("nvc", ('11', "11.4")) - def get_icc_info(self): - return self.gen_compiler_dict("icc", ('2018', "2018.4")) + def _prepare_infrastructure(self) -> None: + """基础设施准备（原子化操作）""" + if self.IS_NORMAL and self._is_first_install: + self.init_linkfile() + if self._is_first_install: + for dir_item in self._paths.values(): + os.makedirs(dir_item, mode=0o755, exist_ok=True) - def get_hmpi_version(self, hmpi_v3_info): - if hmpi_v3_info != "": - ucg_path = self.get_cmd_output('which ucg_info')[0] - else: - ucg_path = self.get_cmd_output('which ucx_info')[0] - ver_dict = {('2','2.0.0'): ('1','1.3.0')} - ucg_path = os.path.dirname(ucg_path) - ucg_path = os.path.dirname(ucg_path) - libucg_path = os.path.join(ucg_path, "lib") - libucg_so_flag = "libucg.so." - version = None - for file_name in os.listdir(libucg_path): - if libucg_so_flag in file_name: - version = self.get_version_info(file_name) - if version in ver_dict: - return ver_dict[version] - elif version: - break - return version - - def get_hmpi_info(self): - hmpi_v2_info = (self.get_cmd_output('ucx_info -c | grep -i BUILT')[0]).upper() - hmpi_v3_info = (self.get_cmd_output('ucg_info -c | grep -i PLANC')[0]).upper() - if "BUILT" not in hmpi_v2_info and "PLANC" not in hmpi_v3_info: - return None - name = 'hmpi' - version = self.get_hmpi_version(hmpi_v3_info) - return self.gen_mpi_dict(name, version) - - def get_openmpi_info(self): - mpi_info_list = self.get_cmd_output('mpirun -version') - mpi_info = mpi_info_list[0].strip() - name = 'openmpi' - version = self.get_version_info(mpi_info) - if not version: - return None - return self.gen_mpi_dict(name, version) - - def get_mpich_info(self): - mpi_info_list = self.get_cmd_output('mpirun -version') - mpi_info = "".join(mpi_info_list).strip() - name = 'mpich' - if name not in mpi_info: - return None - version = self.get_version_info(mpi_info) - if not version: - return None - return self.gen_mpi_dict(name, version) + def init_linkfile(self): + linkfile_content = ''' +#!/bin/bash + +# 源目录数组 +SOURCEDIRS=("compiler/" "lib/" "misc/" "mpi/" "tool/" "app/") + +# 目标目录 +TARGETDIR="modules/" + +# 确保目标目录存在 +mkdir -p "$TARGETDIR" + +# 函数用于递归遍历目录并创建软链接 +link_files() { + local SOURCEDIR="$1" + local TARGETDIR="$2" + # 遍历源目录中的所有文件和子目录 + for FILE in "$SOURCEDIR"*; do + # 检查是否为目录 + if [ -d "$FILE" ] && [ "$FILE" != "$SOURCEDIR"." ] && [ "$FILE" != "$SOURCEDIR".." ]; then + # 创建软链接 + SUBDIRNAME=$(basename "$FILE") + rel_path=$(realpath "$FILE") + ln -sfn "$rel_path" "$TARGETDIR/$SUBDIRNAME" + fi + done +} + +# 遍历所有源目录 +for SOURCEDIR in "${SOURCEDIRS[@]}"; do + link_files "$SOURCEDIR" "$TARGETDIR" +done +''' + self.tool.write_file(self.LINK_FILE, linkfile_content) def get_mpi_info(self): - mpich_info = self.get_mpich_info() - if mpich_info: - return mpich_info - hmpi_info = self.get_hmpi_info() - if hmpi_info: - return hmpi_info - openmpi_info = self.get_openmpi_info() - if openmpi_info: - return openmpi_info + mpich_info = self.mpich_detector.detect() + if mpich_info:return mpich_info + hmpi_info = self.hmpi_detector.detect() + if hmpi_info:return hmpi_info + openmpi_info = self.openmpi_detector.detect() + if openmpi_info:return openmpi_info print("MPI not found, please install MPI first.") sys.exit() @@ -162,7 +172,7 @@ class InstallService: return abs_software_path def check_compiler_mpi(self, compiler_list, compiler_mpi_info): - no_compiler = ["COM","ANY"] + no_compiler = ["COM","ANY","MISC","APP"] is_valid = False compiler_mpi_info = compiler_mpi_info.upper() valid_list = [] @@ -189,6 +199,10 @@ class InstallService: return SType.COMPILER elif compiler_mpi_info == "ANY": return SType.UTIL + elif compiler_mpi_info == "MISC": + return SType.MISC + elif compiler_mpi_info == "APP": + return SType.APP else: return SType.LIB @@ -197,12 +211,15 @@ class InstallService: return software_info_list[2] return "" - def get_software_info(self, software_path, compiler_mpi_info): + def get_software_info(self, software_path, compiler_mpi_info, isapp = False): software_info_list = software_path.split('/') software_name = software_info_list[0] software_version = software_info_list[1] software_main_version = self.get_main_version(software_version) - software_type = self.get_software_type(software_name, compiler_mpi_info) + if isapp: + software_type = SType.APP + else: + software_type = self.get_software_type(software_name, compiler_mpi_info) software_info = { "sname":software_name, "sversion": software_version, @@ -210,7 +227,7 @@ class InstallService: "type" : software_type, "suffix": self.get_suffix(software_info_list) } - if software_type == SType.LIB or software_type == SType.MPI: + if software_type == SType.LIB or software_type == SType.MPI or software_type == SType.APP: software_info["is_use_mpi"] = self.is_contained_mpi(compiler_mpi_info) software_info["use_compiler"] = self.get_used_compiler(compiler_mpi_info) return software_info @@ -241,7 +258,10 @@ class InstallService: return False mpi_str = mpi_info["name"]+mpi_info[self.FULL_VERSION] print("Use MPI: "+mpi_str) - install_path = os.path.join(install_path, mpi_str) + if self.IS_PRO: + install_path = os.path.join(install_path, mpi_str) + elif self.IS_NORMAL: + install_path = f"{install_path}-{mpi_str}" return install_path def get_install_path(self, software_info, env_info): @@ -254,17 +274,31 @@ class InstallService: software_info['sname'] += '-' + suffix sname = software_info['sname'] if stype == SType.MPI: - return os.path.join(self.MPI_PATH, f"{sname}{sversion}-{cname}{cfullver}", sversion) + if self.IS_PRO: + mpi_path = os.path.join(self.MPI_PATH, f"{sname}{sversion}-{cname}{cfullver}", sversion) + if self.IS_NORMAL: + mpi_path = os.path.join(self.MPI_PATH,sname, f"{sversion}-{cname}{cfullver}") + print(f"mpi_path:{mpi_path}") + return mpi_path if stype == SType.COMPILER: install_path = os.path.join(self.COMPILER_PATH, f'{sname}/{sversion}') elif stype == SType.UTIL: install_path = os.path.join(self.UTILS_PATH, f'{sname}/{sversion}') - else: - # install library - install_path = os.path.join(self.LIBS_PATH, cname+cfullver) - # get mpi name and version + elif stype == SType.MISC: + install_path = os.path.join(self.MISC_PATH, f'{sname}/{sversion}') + elif stype == SType.APP: + install_path = os.path.join(self.APP_PATH, f'{sname}/{sversion}-{cname}{cfullver}') install_path = self.add_mpi_path(software_info, install_path) - install_path = os.path.join(install_path, f'{sname}/{sversion}') + else: + if self.IS_PRO: + # install library + install_path = os.path.join(self.LIBS_PATH, cname+cfullver) + # get mpi name and version + install_path = self.add_mpi_path(software_info, install_path) + install_path = os.path.join(install_path, f'{sname}/{sversion}') + elif self.IS_NORMAL: + install_path = os.path.join(self.LIBS_PATH, f'{sname}/{sversion}-{cname}{cfullver}') + install_path = self.add_mpi_path(software_info, install_path) return install_path def is_contained_mpi(self, compiler_mpi_info): @@ -287,8 +321,45 @@ class InstallService: libs_dir.append(os.path.join(prefix, "lib/kspblas/multi")) return libs_dir + def get_loaded_modules(self): + import subprocess + try: + # 执行module list命令(需初始化module环境) + cmd = "source /etc/profile.d/modules.sh; module list 2>&1" + output = subprocess.check_output(cmd, shell=True, executable='/bin/bash') + raw_list = output.decode().split() + modules = [] + for item in raw_list: + if '/' in item: + modules.append(item) + return modules + except subprocess.CalledProcessError as e: + print(f"module list执行失败: {e}") + return [] + + def complement_dep(self): + loaded_modules = self.get_loaded_modules() + print(f"loaded: {loaded_modules}") + deps = self.dependencies.split() + depsmap = {} + for dep in deps: + depsmap[dep] = False + for loaded_module in loaded_modules: + if loaded_module.startswith(dep): + depsmap[dep] = loaded_module + print(f"{loaded_module} for dependency {dep} loaded.") + if not depsmap[dep]: + print(f"{dep} not loaded, please use module load first.") + sys.exit(0) + return " ".join(depsmap.values()) + def get_module_file_content(self, install_path, sname, sversion): module_file_content = '' + if "hpckit" in sname.lower(): + module_file_content = f'''#%Module1.0##################################################################### +prepend-path MODULEPATH {install_path}/HPCKit/latest/modulefiles +''' + return module_file_content file_list = self.get_files(install_path) bins_dir_type = ["bin"] libs_dir_type = ["libs", "lib", "lib64"] @@ -328,10 +399,20 @@ class InstallService: if self.is_mpi_software(sname): opal_prefix = f"setenv OPAL_PREFIX {install_path}" pmix_install_prefix = f"setenv PMIX_INSTALL_PREFIX {install_path}" + depend_content='' + if self.dependencies: + # complement dependencies + depend_content=''' +foreach dep {%s} { + if { ![is-loaded $dep] } { + module load $dep + } +} +''' % self.dependencies module_file_content = f'''#%Module1.0##################################################################### set prefix {install_path} -set version {sversion} - +set version {sversion} +{depend_content} setenv {sname.upper().replace('-','_')}_PATH {install_path} {compiler_values} {opal_prefix} @@ -347,7 +428,7 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} installed_file_path = os.path.join(install_path, "installed") if self.tool.read_file(installed_file_path) == "1": return True - return self.json.query_data(install_path) + return self.json.get(install_path) def gen_module_file(self, install_path, software_info, env_info): sname = software_info['sname'] @@ -365,41 +446,114 @@ setenv {sname.upper().replace('-','_')}_PATH {install_path} if stype == SType.MPI: compiler_str = cname + cfullversion software_str = sname + sversion - module_path = os.path.join(self.MODULE_DEPS_PATH, compiler_str ,sname) - attach_module_path = os.path.join(self.MODULE_DEPS_PATH, compiler_str+'-'+software_str) - self.tool.mkdirs(attach_module_path) - module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" - print(f'attach module file {attach_module_path} successfully generated.') - else: - if stype == SType.COMPILER: - software_str = sname + sversion - module_path = os.path.join(self.MODULE_FILES, sname) - attach_module_path = os.path.join(self.MODULE_DEPS_PATH, software_str) + if self.IS_PRO: + module_path = os.path.join(self.MODULE_DEPS_PATH, compiler_str ,sname) + attach_module_path = os.path.join(self.MODULE_DEPS_PATH, compiler_str+'-'+software_str) self.tool.mkdirs(attach_module_path) module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" print(f'attach module file {attach_module_path} successfully generated.') + elif self.IS_NORMAL: + module_path = os.path.join(self.MODULE_COMPILER_PATH, sname, f'{sversion}-{compiler_str}') + else: + if stype == SType.COMPILER: + software_str = sname + sversion + if self.IS_PRO: + module_path = os.path.join(self.MODULE_FILES, sname) + attach_module_path = os.path.join(self.MODULE_DEPS_PATH, software_str) + self.tool.mkdirs(attach_module_path) + module_file_content += f"\nprepend-path MODULEPATH {attach_module_path}" + print(f'attach module file {attach_module_path} successfully generated.') + elif self.IS_NORMAL: + module_path = os.path.join(self.MODULE_COMPILER_PATH, sname, sversion) elif stype == SType.UTIL: - module_path = os.path.join(self.MODULE_FILES, sname) + if self.IS_PRO: + module_path = os.path.join(self.MODULE_FILES, sname) + elif self.IS_NORMAL: + module_path = os.path.join(self.MODULE_TOOL_PATH, sname, sversion) + elif stype == SType.MISC: + if self.IS_PRO: + module_path = os.path.join(self.MODULE_FILES, sname) + elif self.IS_NORMAL: + module_path = os.path.join(self.MODULE_MISC_PATH, sname, sversion) else: + if self.IS_NORMAL: + if stype == SType.APP: + BASE_PATH = self.MODULE_APP_PATH + else: + BASE_PATH = self.MODULE_LIB_PATH + elif self.IS_PRO: + if stype == SType.APP: + BASE_PATH = self.MODULE_APP_PATH + else: + BASE_PATH = self.MODULE_DEPS_PATH compiler_str = cname + cfullversion if software_info['is_use_mpi']: mpi_info = self.get_mpi_info() mpi_str = mpi_info['name'] + mpi_info[self.FULL_VERSION] - module_path = os.path.join(self.MODULE_DEPS_PATH, f"{compiler_str}-{mpi_str}" ,sname) + if self.IS_PRO: + module_path = os.path.join(self.BASE_PATH, f"{compiler_str}-{mpi_str}" ,sname) + if self.IS_NORMAL: + module_path = os.path.join(BASE_PATH, sname, f"{sversion}-{compiler_str}-{mpi_str}") else: - module_path = os.path.join(self.MODULE_DEPS_PATH, compiler_str, sname) - self.tool.mkdirs(module_path) - module_file = os.path.join(module_path, sversion) + if self.IS_PRO: + module_path = os.path.join(self.BASE_PATH, compiler_str, sname) + elif self.IS_NORMAL: + module_path = os.path.join(BASE_PATH, sname, f"{sversion}-{compiler_str}") + if self.IS_PRO: + self.tool.mkdirs(module_path) + module_file = os.path.join(module_path, sversion) + elif self.IS_NORMAL: + self.tool.mkdirs(os.path.dirname(module_path)) + module_file = module_path self.tool.write_file(module_file, module_file_content) print(f"module file {module_file} successfully generated") - row = self.json.query_data(install_path) + row = self.json.get(install_path) row["module_path"] = module_file - self.json.update_data(install_path, row) - self.json.write_file() + self.json.set(install_path, row, True) + #更新linkfile + if self.IS_NORMAL:self.update_modules() + + def update_modules(self): + upd_cmd = f''' +cd {self.MODULE_FILES} +source {self.LINK_FILE} +''' + result = self.exe.exec_raw(upd_cmd) + if result: + print(f"update modules successful") + else: + print("update failed") + + def extract_dependency_variable(self,file_path): + """ + 从 install.sh 文件中提取环境变量的值 + :param file_path: install.sh 文件路径 + :return: extract 的值（字符串），未找到时返回 None + """ + ex_value = None + # 正则匹配 export AA=value 或 AA=value 的行（忽略注释和空格） + pattern = re.compile(r'^\s*(?:export\s+)?DEPENDENCIES\s*=\s*(.*?)(?:\s*#.*)?$', re.IGNORECASE) + + with open(file_path, 'r') as f: + for line in f: + line = line.strip() + if line.startswith('#'): + continue # 跳过注释行 + match = pattern.match(line) + if match: + value = match.group(1) + # 去除两侧的引号（单引号、双引号） + if value.startswith(('"', "'")) and value.endswith(('"', "'")): + value = value[1:-1] + ex_value = value + return ex_value def install_package(self, abs_software_path, install_path, other_args): install_script = 'install.sh' install_script_path = os.path.join(abs_software_path, install_script) + # Extracting dependency information + self.dependencies = self.extract_dependency_variable(install_script_path) + if self.dependencies:self.dependencies = self.complement_dep() print("start installing..."+ abs_software_path) if not os.path.exists(install_script_path): print("install script not exists, skipping...") @@ -430,34 +584,35 @@ chmod +x {install_script} software_dict['name'] = software_info['sname'] software_dict['version'] = software_info['sversion'] software_dict['module_path'] = '' - self.json.add_data(install_path, software_dict) - self.json.write_file() + self.json.set(install_path, software_dict, True) def remove_prefix(self, software_path): if software_path.startswith('package/') or software_path.startswith('./'): software_path = software_path.replace('./', '', 1) software_path = software_path.replace('package/', '', 1) return software_path - - def install(self, install_args): + + def install(self, install_args, isapp = False): software_path = install_args[0] compiler_mpi_info = install_args[1] other_args = install_args[2:] self.tool.prt_content("INSTALL " + software_path) - compilers = {"GCC":self.get_gcc_info, "CLANG":self.get_clang_info, - "NVC":self.get_nvc_info, "ICC":self.get_icc_info, - "BISHENG":self.get_clang_info} + + compilers = {"GCC":self.gcc_detector.detect, "CLANG":self.clang_detector.detect, + "NVC":self.nvcc_detector.detect, "ICC":self.icc_detector.detect, + "BISHENG": self.clang_detector.detect} software_path = self.remove_prefix(software_path) # software_path should exists - abs_software_path = self.check_software_path(software_path) - if not abs_software_path: return + if not isapp: + abs_software_path = self.check_software_path(software_path) + if not abs_software_path: return compiler_mpi_info = self.check_compiler_mpi(compilers.keys(), compiler_mpi_info) if not compiler_mpi_info: return - software_info = self.get_software_info(software_path, compiler_mpi_info) + software_info = self.get_software_info(software_path, compiler_mpi_info, isapp) stype = software_info['type'] # get compiler name and version env_info = self.get_compiler_info(compilers, compiler_mpi_info) - if stype == SType.LIB or stype == SType.MPI: + if stype == SType.LIB or stype == SType.MPI or stype == SType.APP: cmversion = env_info['cmversion'] cfullver = env_info[self.FULL_VERSION] if cmversion == None: @@ -468,7 +623,16 @@ chmod +x {install_script} # get install path install_path = self.get_install_path(software_info, env_info) - if not install_path: return + if not install_path: + return + else: + self.tool.mkdirs(install_path) + if isapp: + return { + "install_path": install_path, + "software_info":software_info, + "env_info":env_info + } # get install script self.install_package(abs_software_path, install_path, other_args) # add install info @@ -477,23 +641,20 @@ chmod +x {install_script} self.gen_module_file( install_path, software_info, env_info) def install_depend(self): - depend_file = 'depend_install.sh' - print(f"start installing dependendcy of {DataService.app_name}") - depend_content = f''' -source ./init.sh -{DataService.dependency} -''' + depend_file = self.ds.get_depend_file() + print(f"start installing dependendcy of {self.ds.app_config.name}") + depend_content = f'''{self.ds.get_dependency()}''' self.tool.write_file(depend_file, depend_content) run_cmd = f''' chmod +x {depend_file} -./{depend_file} +sh {depend_file} ''' self.exe.exec_raw(run_cmd) def remove(self, software_info): self.tool.prt_content("UNINSTALL " + software_info) remove_list = [] - installed_dict = self.json.read_file() + installed_dict = self.json.data for path, software_row in installed_dict.items(): if software_info in software_row['name']: remove_list.append((path, software_row)) @@ -514,13 +675,13 @@ chmod +x {depend_file} return except: sys.exit("please enter a valid number!") - self.json.delete_data(remove_list[choice-1][0]) - self.json.write_file() + self.json.delete(remove_list[choice-1][0], True) print("Successfully remove "+software_info) def list(self): self.tool.prt_content("Installed list".upper()) - installed_list = self.json.read_file() + installed_list = self.json.data + installed_list.pop('MODE', None) if len(installed_list) == 0: print("no software installed.") return @@ -544,7 +705,7 @@ chmod +x {depend_file} def find(self, content): self.tool.prt_content(f"Looking for package {content}") - installed_list = list(self.json.read_file().values()) + installed_list = list(self.json.data.values()) for row in installed_list: if content in row['name']: print(row) diff --git a/src/installStrategy.py b/src/installStrategy.py new file mode 100644 index 0000000000000000000000000000000000000000..344973ed7f3eb3f16a61aafed5f965eaeb54f983 --- /dev/null +++ b/src/installStrategy.py @@ -0,0 +1,56 @@ +import os +from pathlib import Path +from typing import Dict, Type +from installTypes import InstallMode +class PathStrategy: + paths = { + 'SOFTWARE_PATH': 'JARVIS_SOFT_ROOT', + 'COMPILER_PATH': 'JARVIS_COMPILER', + 'LIBS_PATH': 'JARVIS_LIBS', + 'MODULE_FILES': 'JARVIS_MODULES', + 'MPI_PATH': 'JARVIS_MPI', + 'UTILS_PATH': 'JARVIS_UTILS', + 'MISC_PATH': 'JARVIS_MISC', + 'APP_PATH': 'JARVIS_APP', + 'MODULE_APP_PATH': 'JARVIS_MODULES_APP' + } + def get_paths(self) -> Dict[str, str]: + raise NotImplementedError + +class ProPathStrategy(PathStrategy): + def get_paths(self) -> Dict[str, str]: + merged_paths = self.paths.copy() + merged_paths.update({ + "MODULE_DEPS_PATH":"JARVIS_MODULEDEPS" + }) + return {k: os.getenv(v) for k, v in merged_paths.items()} + +class NormalPathStrategy(PathStrategy): + def get_paths(self) -> Dict[str, str]: + merged_paths = self.paths.copy() + merged_paths.update({ + "MODULE_LIB_PATH":"JARVIS_MODULES_LIB", + 'MODULE_COMPILER_PATH': 'JARVIS_MODULES_COMPILER', + 'MODULE_MISC_PATH': 'JARVIS_MODULES_MISC', + 'MODULE_MPI_PATH': 'JARVIS_MODULES_MPI', + 'MODULE_MOD_PATH': 'JARVIS_MODULES_MODS', + 'JOBSCRIPT_PATH': 'JARVIS_JOBSCRIPT' + }) + return {k: os.getenv(v) for k, v in merged_paths.items()} + +class PathStrategyFactory: + @classmethod + def create(cls, mode: InstallMode) -> Type[PathStrategy]: + strategies = { + InstallMode.PRO: ProPathStrategy, + InstallMode.NORMAL: NormalPathStrategy + } + return strategies[mode]() + +if __name__ == "__main__": + service = PathStrategyFactory() + mode = service.create(InstallMode.PRO) + print(mode.get_paths()) + mode = service.create(InstallMode.NORMAL) + print(mode.get_paths()) + diff --git a/src/installTypes.py b/src/installTypes.py new file mode 100644 index 0000000000000000000000000000000000000000..3bbf56a0d31490c1f4ccdd84cd564ae59b6d0c37 --- /dev/null +++ b/src/installTypes.py @@ -0,0 +1,18 @@ +# types.py +from enum import Enum +from typing import Tuple, Optional, Dict + +class InstallMode(Enum): + PRO = "0" + NORMAL = "1" + +class SoftwareCategory(Enum): + COMPILER = 1 + MPI = 2 + UTIL = 3 + LIB = 4 + MISC = 5 + APP = 6 + +VersionInfo = Tuple[str, str] # (major_version, full_version) + diff --git a/src/jarvis.py b/src/jarvis.py index 24404709e0dd16eddaf883b4e7b632a6f13de24e..3808cd676bdc586a78421e2991c08d5e546a0b00 100644 --- a/src/jarvis.py +++ b/src/jarvis.py @@ -1,15 +1,18 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse - +from initService import InitService from dataService import DataService from analysisService import AnalysisService class Jarvis: def __init__(self): + self.init = InitService() self.analysis = AnalysisService() + self.ds = DataService() + self.app_name = self.ds.get_app_name() # Argparser set - parser = argparse.ArgumentParser(description=f'please put me into CASE directory, used for {DataService.app_name} Compiler/Clean/Run/Compare', + parser = argparse.ArgumentParser(description=f'please put me into CASE directory, used for {self.app_name} Compiler/Clean/Run/Compare', usage='%(prog)s [-h] [--build] [--clean] [...]') parser.add_argument("-v","--version", help=f"get version info", action="store_true") parser.add_argument("-use","--use", help="Switch config file...", nargs=1) @@ -23,15 +26,15 @@ class Jarvis: #find parser.add_argument("-find","--find", help=f"find software", nargs=1) # dependency install - parser.add_argument("-dp","--depend", help=f"{DataService.app_name} dependency install", action="store_true") - parser.add_argument("-e","--env", help=f"set environment {DataService.app_name}", action="store_true") - parser.add_argument("-b","--build", help=f"compile {DataService.app_name}", action="store_true") - parser.add_argument("-cls","--clean", help=f"clean {DataService.app_name}", action="store_true") - parser.add_argument("-r","--run", help=f"run {DataService.app_name}", action="store_true") - parser.add_argument("-j","--job", help=f"run job {DataService.app_name}", action="store_true") - parser.add_argument("-j2","--job2", help=f"run job 2 {DataService.app_name}", action="store_true") - parser.add_argument("-p","--perf", help=f"auto perf {DataService.app_name}", action="store_true") - parser.add_argument("-kp","--kperf", help=f"auto kperf {DataService.app_name}", action="store_true") + parser.add_argument("-dp","--depend", help=f"{self.app_name} dependency install", action="store_true") + parser.add_argument("-e","--env", help=f"set environment {self.app_name}", action="store_true") + parser.add_argument("-b","--build", help=f"compile {self.app_name}", action="store_true") + parser.add_argument("-cls","--clean", help=f"clean {self.app_name}", action="store_true") + parser.add_argument("-r","--run", help=f"run {self.app_name}", action="store_true") + parser.add_argument("-j","--job", help=f"run job {self.app_name}", action="store_true") + parser.add_argument("-j2","--job2", help=f"run job 2 {self.app_name}", action="store_true") + parser.add_argument("-p","--perf", help=f"auto perf {self.app_name}", action="store_true") + parser.add_argument("-kp","--kperf", help=f"auto kperf {self.app_name}", action="store_true") # GPU perf parser.add_argument("-gp","--gpuperf", help="GPU perf...", action="store_true") # hpctool perf @@ -39,9 +42,9 @@ class Jarvis: # NCU perf parser.add_argument("-ncu","--ncuperf", help="NCU perf...", nargs=1) - parser.add_argument("-c","--compare", help=f"compare {DataService.app_name}", nargs=2) + parser.add_argument("-c","--compare", help=f"compare {self.app_name}", nargs=2) # batch run - parser.add_argument("-rb","--rbatch", help=f"run batch {DataService.app_name}", action="store_true") + parser.add_argument("-rb","--rbatch", help=f"run batch {self.app_name}", action="store_true") # batch download parser.add_argument("-d","--download", help="Batch Download...", action="store_true") # generate singularity def file diff --git a/src/jsonService.py b/src/jsonService.py index 24835c0a83882be57fd7d12f4e7a3968179b9e7e..4f7e05de384ebf3f8bc68057367ab19fd01a2e7c 100644 --- a/src/jsonService.py +++ b/src/jsonService.py @@ -1,49 +1,124 @@ import json import os +from pathlib import Path +from typing import Any, Dict, Optional class JSONService: - def __init__(self, filename): - self.filename = filename - self.data = self.read_file() - - # 读取 JSON 文件 - def read_file(self): - if not os.path.exists(self.filename): - with open(self.filename, 'w') as f: - f.write('{}') - with open(self.filename, "r") as file: - data = json.load(file) - return data - - # 写入 JSON 文件 - def write_file(self): - with open(self.filename, "w") as file: - json.dump(self.data, file, indent=4) - - # 查询数据 - def query_data(self, key): - if key in self.data: - return self.data[key] - else: - return None - - # 添加数据 - def add_data(self, key, value): - self.data[key] = value - - # 删除数据 - def delete_data(self, key): - if key in self.data: - del self.data[key] - else: - print("Key not found") - - # 修改数据 - def update_data(self, key, value): - if key in self.data: - self.data[key] = value - else: - print("Key not found") - - def json_transform(self, dict): - return json.dumps(dict) \ No newline at end of file + """提供JSON文件读写和数据处理的服务类""" + + def __init__(self, filepath: str) -> None: + """ + 初始化JSON服务 + + :param filepath: JSON文件路径 + :raises FileNotFoundError: 当文件不存在时抛出 + :raises json.JSONDecodeError: 当JSON解析失败时抛出 + """ + self._filepath = Path(filepath).absolute() + self._data: Dict[str, Any] = {} + self._load_data() + + def _load_data(self) -> None: + """加载JSON文件数据到内存""" + try: + if not self._filepath.exists(): + raise FileNotFoundError(f"JSON文件不存在: {self._filepath}") + + with self._filepath.open('r', encoding='utf-8') as f: + self._data = json.load(f) + + if not isinstance(self._data, dict): + raise ValueError("JSON文件内容必须为字典结构") + + except json.JSONDecodeError as e: + raise json.JSONDecodeError(f"JSON解析失败: {self._filepath}", e.doc, e.pos) from e + + def save(self, indent: int = 4, ensure_ascii: bool = False) -> None: + """ + 将内存数据持久化到文件 + + :param indent: 缩进空格数 + :param ensure_ascii: 是否转义非ASCII字符 + """ + with self._filepath.open('w', encoding='utf-8') as f: + json.dump(self._data, f, indent=indent, ensure_ascii=ensure_ascii) + + @property + def data(self) -> Dict[str, Any]: + """获取完整数据字典的只读视图""" + return self._data.copy() + + def get(self, key: str, default: Optional[Any] = None) -> Optional[Any]: + """ + 安全获取数据项 + + :param key: 数据键名 + :param default: 键不存在时返回的默认值 + :return: 键对应的值或默认值 + """ + return self._data.get(key, default) + + def set(self, key: str, value: Any, auto_save: bool = False) -> None: + """ + 设置数据项 + + :param key: 数据键名 + :param value: 要设置的值 + :param auto_save: 是否自动保存到文件 + :raises TypeError: 当键名不是字符串时抛出 + """ + if not isinstance(key, str): + raise TypeError("键名必须是字符串类型") + self._data[key] = value + if auto_save: + self.save() + + def delete(self, key: str, auto_save: bool = False) -> bool: + """ + 删除数据项 + + :param key: 要删除的键名 + :param auto_save: 是否自动保存到文件 + :return: 是否成功删除 + """ + if key in self._data: + del self._data[key] + if auto_save: + self.save() + return True + return False + + def update(self, mapping: Dict[str, Any], auto_save: bool = False) -> None: + """ + 批量更新数据 + + :param mapping: 要合并的字典数据 + :param auto_save: 是否自动保存到文件 + """ + self._data.update(mapping) + if auto_save: + self.save() + + @classmethod + def from_dict(cls, data: Dict[str, Any], filepath: str) -> 'JSONService': + """ + 从字典创建JSON服务实例 + + :param data: 初始数据字典 + :param filepath: 目标文件路径 + :return: JSONService实例 + """ + instance = cls(filepath) + instance._data = data.copy() + instance.save() + return instance + + def to_json_str(self, indent: Optional[int] = None) -> str: + """ + 将当前数据转换为JSON字符串 + + :param indent: 缩进格式设置 + :return: 格式化后的JSON字符串 + """ + return json.dumps(self._data, indent=indent, ensure_ascii=False) + diff --git a/src/perfService.py b/src/perfService.py index 5952a94e33c0b6511518d4678dea4e92bef4283f..797db55f6a610c78f62ef53e6bf99c2e60b1a454 100644 --- a/src/perfService.py +++ b/src/perfService.py @@ -6,21 +6,23 @@ import re import os from glob import glob +from commandBuilder import CommandBuilder from dataService import DataService from executeService import ExecuteService from toolService import ToolService class PerfService: def __init__(self): - self.hpc_data = DataService() + self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() self.ROOT = os.getcwd() + self.command = CommandBuilder() self.isARM = platform.machine() == 'aarch64' def get_pid(self): #get pid - pid_cmd = f'pidof {DataService.binary_file}' + pid_cmd = f'pidof {self.ds.get_binary_file()}' result = self.exe.exec_popen(pid_cmd) if len(result) == 0: print("failed to get pid.") @@ -31,19 +33,19 @@ class PerfService: return pid_list[mid].strip() def perf(self): - print(f"start perf {DataService.app_name}") + print(f"start perf {self.ds.app_config.name}") #get pid pid = self.get_pid() #start perf && analysis perf_cmd = f''' yum install -y perf -perf record {DataService.perf_para} -a -g -p {pid} +perf record {self.perf_config.perf} -a -g -p {pid} perf report -i ./perf.data -F period,sample,overhead,symbol,dso,comm -s overhead --percent-limit 0.1% --stdio ''' self.exe.exec_raw(perf_cmd) def kperf(self): - print(f"start kperf {DataService.app_name}") + print(f"start kperf {self.ds.app_config.name}") python3_libs_path = './software/libs/python3/' #get pid pid = self.get_pid() @@ -52,7 +54,7 @@ perf report -i ./perf.data -F period,sample,overhead,symbol,dso,comm -s overhea #start kperf kperf_cmd = f''' chmod +x {kperf_script} -python3 {kperf_script} --rawdata --hotfunc --topdown --cache --tlb --imix {DataService.kperf_para} --duration 1 --interval 15 --pid {pid} {kperf_log} +python3 {kperf_script} --rawdata --hotfunc --topdown --cache --tlb --imix {self.ds.perf_config.kperf_para} --duration 1 --interval 15 --pid {pid} {kperf_log} ''' self.exe.exec_raw(kperf_cmd) print("kperf.data.txt is generated") @@ -68,8 +70,8 @@ python3 {kperf_script} --rawdata --hotfunc --topdown --cache --tlb --imix {DataS def gpu_perf(self): print("start gpu perf") - run_cmd = self.hpc_data.get_run() - env_cmd = self.hpc_data.get_env() + run_cmd = self.command.full_run() + env_cmd = self.command.env_activation() nsys_para = '-y 5s -d 100s' if DataService.nsys_para: nsys_para = DataService.nsys_para @@ -135,8 +137,8 @@ ncu --export ncu-{self.get_arch()}-{self.get_cur_time()} --import-source=yes --s #collect hpccollect_path = f'$JARVIS_UTILS/hpctool/{max_version}/bin/hpccollect' hpccollect_para = f' -l detail -o ./{output_file}' - if DataService.hpccollect_para != '': - hpccollect_para = DataService.hpccollect_para + hpccollect_para + if self.ds.perf_config.hpccollect != '': + hpccollect_para = self.ds.perf_config.hpccollect + hpccollect_para collect_cmd = self.hpc_data.get_run(f'{hpccollect_path} {hpccollect_para}') #analysis hpcreport_path = f'$JARVIS_UTILS/hpctool/{max_version}/bin/hpcreport' @@ -144,15 +146,15 @@ ncu --export ncu-{self.get_arch()}-{self.get_cur_time()} --import-source=yes --s hpcreport_para = f'-i ./{output_file}.v{max_version}.hpcstat hotspots -g parallel-region function' else: hpcreport_para = f'-i ./{output_file}.v{max_version}.hpcstat mpi-wait -g function' - if DataService.hpcreport_para != '': - hpcreport_para = DataService.hpcreport_para + if self.ds.perf_config.hpcreport != '': + hpcreport_para = self.ds.perf_config.hpcreport report_cmd = f'{hpcreport_path} {hpcreport_para}' collect_and_report_cmd = f''' {self.hpc_data.get_env()} ./jarvis -install hpctool/{max_version} any echo "1">/proc/sys/kernel/perf_event_paranoid -cd {DataService.case_dir} +cd {self.ds.app_config.case_dir} {collect_cmd} {report_cmd} ''' - self.exe.exec_raw(collect_and_report_cmd) \ No newline at end of file + self.exe.exec_raw(collect_and_report_cmd) diff --git a/src/runService.py b/src/runService.py index e58209e8697f4fa1c17a03e233e61f864416b00d..5e3f634a58af94114084635005f01cfd7b100444 100644 --- a/src/runService.py +++ b/src/runService.py @@ -5,14 +5,16 @@ import sys from toolService import ToolService from executeService import ExecuteService from dataService import DataService +from commandBuilder import CommandBuilder class RunService: def __init__(self): - self.hpc_data = DataService() + self.ds = DataService() self.exe = ExecuteService() self.tool = ToolService() + self.command = CommandBuilder() # 注入命令生成组件 self.ROOT = os.getcwd() - self.avail_ips_list = self.tool.gen_list(DataService.avail_ips) + self.avail_ips_list = self.tool.gen_list(self.ds.get_available_ips()) def gen_hostfile(self, nodes): length = len(self.avail_ips_list) @@ -27,52 +29,16 @@ class RunService: # single run def run(self): - print(f"start run {DataService.app_name}") - nodes = int(DataService.run_cmd['nodes']) + print(f"start run {self.ds.get_app_name()}") + nodes = int(self.ds.get_run_cmd('nodes')) self.gen_hostfile(nodes) - run_cmd = self.hpc_data.get_run_cmd() + run_cmd = self.command.full_run() self.exe.exec_raw(run_cmd) def batch_run(self): - batch_file = 'batch_run.sh' - batch_file_path = os.path.join(self.ROOT, batch_file) - print(f"start batch run {DataService.app_name}") - batch_content = f''' -{self.hpc_data.get_env()} -cd {DataService.case_dir} -{DataService.batch_cmd} -''' - self.tool.write_file(batch_file_path, batch_content) - run_cmd = f''' -chmod +x {batch_file} -./{batch_file} -''' + batch_cmd = self.command.batch_run() self.exe.exec_raw(run_cmd) def job_run(self,num): - job_file = 'job_run.sh' - job_file_path = os.path.join(self.ROOT, job_file) - print(f"start job run {DataService.app_name}") - job_cmd = DataService.job_cmd if num == 1 else DataService.job2_cmd - job_content = f''' -{self.hpc_data.get_env()} -cd {DataService.case_dir} -cat > run.sh << \EOF -{job_cmd} -EOF - -chmod +x run.sh -if type djob >/dev/null 2>&1;then - dsub -s run.sh -elif type sbatch >/dev/null 2>&1;then - sbatch run.sh -else - echo "dsub not exists." -fi -''' - self.tool.write_file(job_file_path, job_content) - run_cmd = f''' -chmod +x {job_file} -./{job_file} -''' - self.exe.exec_raw(run_cmd) + job_cmd = self.command.job_run(num) + self.exe.exec_raw(job_cmd) diff --git a/src/softwareTypes.py b/src/softwareTypes.py new file mode 100644 index 0000000000000000000000000000000000000000..8350271f32f2d8059b142db6c80e81d373e32d9e --- /dev/null +++ b/src/softwareTypes.py @@ -0,0 +1,26 @@ +from enum import Enum, auto +from dataclasses import dataclass + +class SoftwareType(Enum): + COMPILER = auto() + MPI = auto() + UTIL = auto() + MISC = auto() + LIB = auto() + APP = auto() + +@dataclass(frozen=True) +class SoftwareProfile: + name: str + full_version: str + software_type: SoftwareType + major_version: str + suffix: str = "" + use_mpi: bool = False + compiler_name: str = None + +@dataclass +class CompilerInfo: + name: str + full_version: str + major_version: str diff --git a/src/toolService.py b/src/toolService.py index 0cb256700c82218b59f7b6c30772456dccc31ae5..eed66ebd9c4d4a9d533cabae4ced62bafb56f66e 100644 --- a/src/toolService.py +++ b/src/toolService.py @@ -38,9 +38,17 @@ class ToolService: def write_file(self, filename, content=""): """ 将内容写入文件。 - """ - with open(filename, 'w') as f: - f.write(content) + """ + # 获取目标目录路径 + dir_path = os.path.dirname(filename) + # 若目录不存在，则递归创建（包括所有父目录） + if dir_path: # 避免根目录（如 file_path 是 "file.txt"） + os.makedirs(dir_path, exist_ok=True) + try: + with open(filename, 'w', encoding='utf-8') as f: + f.write(content) + except Exception as e: + print(f"str(e)") def del_file(self, filepath): """ diff --git a/src/versionParser.py b/src/versionParser.py new file mode 100644 index 0000000000000000000000000000000000000000..0f88b5b2ea24bfe23a603a3fb1f9979d9516b360 --- /dev/null +++ b/src/versionParser.py @@ -0,0 +1,31 @@ +import re +from typing import Optional, Pattern, Dict +from installTypes import VersionInfo + +class VersionParser: + DEFAULT_REGEX: Pattern = re.compile(r'(\d+)\.(\d+)\.(\d+)') + + @classmethod + def parse(cls, info: str, pattern: Pattern = DEFAULT_REGEX) -> Optional[VersionInfo]: + """语义化版本解析""" + match = pattern.search(info) + if not match: + return None + return (match.group(1), f"{match[1]}.{match[2]}.{match[3]}") + + @classmethod + def gen_compiler_profile(cls, name: str, version: VersionInfo) -> Dict: + return { + 'cname': name, + 'cmversion': version[0], + 'full_version': version[1] + } + + @classmethod + def gen_mpi_profile(cls, name: str, version: VersionInfo) -> Dict: + return { + 'name': name, + 'mversion': version[0], + 'full_version': version[1] + } +