diff --git a/.gitignore b/.gitignore index 6211baa62df1f80d551935c3935c45b0c1d09434..2a95443a1ca7562ce1d0e2983c47a8812634678c 100644 --- a/.gitignore +++ b/.gitignore @@ -61,5 +61,6 @@ hostfile tmp downloads/* depend_install.sh +build.sh .meta software/* \ No newline at end of file diff --git a/README.md b/README.md index 5b07db44db5c22e5ff796e50e7a0d8c5eb7eb9d3..cbd233561c7d26b949a04fba73db1434e50c736e 100644 --- a/README.md +++ b/README.md @@ -113,53 +113,59 @@ eg: ``` ./jarvis -install bisheng/2.1.0 com #安装毕晟编译器 -./jarvis -install fftw/3.3.8 gcc+mpi #使用gcc和mpi编译fftw 3.3.8版本 -./jarvis -install openmpi/4.1.2 gcc #使用gcc编译openmpi 4.1.2版本 +./jarvis -install fftw/3.3.8 gcc+mpi #使用当前gcc和mpi编译fftw 3.3.8版本 +./jarvis -install openmpi/4.1.2 gcc #使用当前gcc编译openmpi 4.1.2版本 ``` -5.一键安装所有依赖 +5.一键卸载依赖 + +``` +./jarvis -remove openblas/0.3.18 +``` + +6.一键安装所有依赖 ``` ./jarvis -dp ``` -6.一键生成环境变量(脱离贾维斯运行才需要执行) +7.一键生成环境变量(脱离贾维斯运行才需要执行) ``` ./jarvis -e && source ./env.sh ``` -7.一键编译 +8.一键编译 ``` ./jarvis -b ``` -8.一键运行 +9.一键运行 ``` ./jarvis -r ``` -9.一键性能采集(perf) +10.一键性能采集(perf) ``` ./jarvis -p ``` -10.一键GPU性能采集(需安装nsys、ncu) +11.一键GPU性能采集(需安装nsys、ncu) ``` ./jarvis -gp ``` -11.一键输出服务器信息(包括CPU、网卡、OS、内存等) +12.一键输出服务器信息(包括CPU、网卡、OS、内存等) ``` ./jarvis -i ``` -12.一键服务器性能评测(包括MPI、OMP、P2P等) +13.一键服务器性能评测(包括MPI、OMP、P2P等) ``` ./jarvis -bench all #运行所有benchmark @@ -168,18 +174,24 @@ eg: ./jarvis -bench gemm #运行矩阵运算 benchmark ``` -13.切换配置 +14.切换配置 ``` ./jarvis -use XXX.config ``` -14.其它功能查看（网络检测） +15.其它功能查看（网络检测） ``` ./jarvis -h ``` +16.根据当前配置生成Singularity容器定义文件 + +``` +./jarvis -container docker-hub-address +``` + ### 欢迎贡献 @@ -198,4 +210,8 @@ eg: 请添加openEuler HPC SIG微信群了解更多HPC迁移调优知识 -![微信群](./wechat-group-qr.png) \ No newline at end of file +![微信群](./wechat-group-qr.png) + +### 技术文章 + +揭开HPC应用的神秘面纱：https://zhuanlan.zhihu.com/p/489828346 \ No newline at end of file diff --git a/container/README.md b/container/README.md index 92c7734b72f07997bb233cd97f1f7c980c6cd5b2..a253e811a627af8feaca3c50f821d6391d4dc99b 100644 --- a/container/README.md +++ b/container/README.md @@ -49,14 +49,15 @@ source ./init.sh 5.2 安装singularity ``` -./jarvis -install go/1.18 gcc +./jarvis -install go/1.18 com ./jarvis -install singularity/3.9.6 gcc ``` 5.3 生成QE容器包 ``` -cd container && singularity build openeuler-kgcc9-openmpi4-qe-6.4.sif openeuler-kgcc9-openmpi4-qe-6.4.def +cd container +singularity build openeuler-kgcc9-openmpi4-qe-6.4.sif openeuler-kgcc9-openmpi4-qe-6.4.def ``` 5.4 安装和容器同版本的MPI库 @@ -71,7 +72,7 @@ cp ./templates/qe/6.4/data.qe.container.config ./ ``` cd container -mpirun --allow-run-as-root -x OMP_NUM_THREADS=1 -np 96 singularity exec openeuler-kgcc9-openmpi4-qe-6.4.sif /hpcrunner/q-e-qe-6.4.1/bin/pw.x -input /hpcrunner/workloads/QE/qe-test/test_3.in +mpirun --allow-run-as-root -x OMP_NUM_THREADS=1 -np 96 singularity exec --pwd /hpcrunner/workloads/QE/qe-test openeuler-kgcc9-openmpi4-qe-6.4.sif /hpcrunner/q-e-qe-6.4.1/bin/pw.x -input test_3.in ``` ### 欢迎贡献更多的HPC容器！ diff --git a/container/openeuler-bisheng2-hmpi1-qe-6.4-opt.def b/container/openeuler-bisheng2-hmpi1-qe-6.4-opt.def index fea007d8556f02e4f91a1de1a71e0cb84d0c9b55..016ce488c4f82fc7aaf485b94b47fe5a43dec6ad 100644 --- a/container/openeuler-bisheng2-hmpi1-qe-6.4-opt.def +++ b/container/openeuler-bisheng2-hmpi1-qe-6.4-opt.def @@ -22,8 +22,6 @@ From: openeuler/openeuler cp ./templates/qe/6.4/data.qe.container.opt.config . wget --no-check-certificate https://github.com/QEF/q-e/archive/refs/tags/qe-6.4.1.tar.gz tar -xzvf qe-6.4.1.tar.gz - # Modify the case path - sed -i 's/pseudopotentials\.d/\/hpcrunner\/workloads\/QE\/qe-test\/pseudopotentials\.d/g' ./workloads/QE/qe-test/test_3.in # Switch config ./jarvis -use data.qe.container.opt.config # download dependency diff --git a/container/openeuler-kgcc9-openmpi4-qe-6.4.def b/container/openeuler-kgcc9-openmpi4-qe-6.4.def index 9a57c760ab838270c495ad32303bd5610b7db6ff..ec9533786b51299c92d8075a4d0fe2553cfd3f37 100644 --- a/container/openeuler-kgcc9-openmpi4-qe-6.4.def +++ b/container/openeuler-kgcc9-openmpi4-qe-6.4.def @@ -22,8 +22,6 @@ From: openeuler/openeuler cp ./templates/qe/6.4/data.qe.container.config . wget --no-check-certificate https://github.com/QEF/q-e/archive/refs/tags/qe-6.4.1.tar.gz tar -xzvf qe-6.4.1.tar.gz - # Modify the case path - sed -i 's/pseudopotentials\.d/\/hpcrunner\/workloads\/QE\/qe-test\/pseudopotentials\.d/g' ./workloads/QE/qe-test/test_3.in # Switch config ./jarvis -use data.qe.container.config # download dependency diff --git a/src/analysisService.py b/src/analysisService.py index d6796edced4173c9c031f6bb6a37fd33e9ed9633..6e39645eb2df615cdb5580fd5d83c35465f1a229 100644 --- a/src/analysisService.py +++ b/src/analysisService.py @@ -75,6 +75,9 @@ class AnalysisService: def install(self,software_path, compiler_mpi_info): self.jinstall.install(software_path, compiler_mpi_info) + def remove(self,software_path): + self.jinstall.remove(software_path) + def install_deps(self): self.jinstall.install_depend() diff --git a/src/buildService.py b/src/buildService.py index f82580283b5c490725e3e9f43ceb60f5e79f98f9..91739ce3fa82c8906b97e7cd93008be82864e11e 100644 --- a/src/buildService.py +++ b/src/buildService.py @@ -2,11 +2,13 @@ # -*- coding: utf-8 -*- from dataService import DataService from executeService import ExecuteService +from toolService import ToolService class BuildService: def __init__(self): self.hpc_data = DataService() self.exe = ExecuteService() + self.tool = ToolService() def clean(self): print(f"start clean {DataService.app_name}") @@ -16,4 +18,10 @@ class BuildService: def build(self): print(f"start build {DataService.app_name}") build_cmd = self.hpc_data.get_build_cmd() - self.exe.exec_raw(build_cmd) + build_file = 'build.sh' + self.tool.write_file(build_file, build_cmd) + run_cmd = f''' +chmod +x {build_file} +./{build_file} +''' + self.exe.exec_raw(run_cmd) diff --git a/src/installService.py b/src/installService.py index 62c115fc37d071a7a9050f8fba05a32915f75d10..36b8e81aeef4719825bddb075fa33a327e38f1ea 100644 --- a/src/installService.py +++ b/src/installService.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import os +from secrets import choice import sys import re from enum import Enum @@ -23,12 +24,13 @@ class InstallService: self.tool = ToolService() self.ROOT = os.getcwd() self.PACKAGE_PATH = os.path.join(self.ROOT, 'package') - self.COMPILER_PATH = os.path.join(self.ROOT, 'software/compiler') - self.LIBS_PATH = os.path.join(self.ROOT, 'software/libs') - self.MODULE_DEPS_PATH = os.path.join(self.ROOT, 'software/moduledeps') - self.MODULE_FILES = os.path.join(self.ROOT, 'software/modulefiles') - self.MPI_PATH = os.path.join(self.ROOT, 'software/mpi') - self.UTILS_PATH = os.path.join(self.ROOT, 'software/utils') + self.SOFTWARE_PATH = os.path.join(self.ROOT, 'software') + self.COMPILER_PATH = os.path.join(self.SOFTWARE_PATH, 'compiler') + self.LIBS_PATH = os.path.join(self.SOFTWARE_PATH, 'libs') + self.MODULE_DEPS_PATH = os.path.join(self.SOFTWARE_PATH, 'moduledeps') + self.MODULE_FILES = os.path.join(self.SOFTWARE_PATH, 'modulefiles') + self.MPI_PATH = os.path.join(self.SOFTWARE_PATH, 'mpi') + self.UTILS_PATH = os.path.join(self.SOFTWARE_PATH, 'utils') def get_version_info(self, info): return re.search( r'(\d+)\.(\d+)\.',info).group(1) @@ -245,9 +247,9 @@ set version {sversion} return False return True - def set_installed_status(self, install_path): + def set_installed_status(self, install_path, flag="1"): installed_file_path = self.get_installed_file_path(install_path) - self.tool.write_file(installed_file_path, "1") + self.tool.write_file(installed_file_path, flag) def gen_module_file(self, install_path, software_info, env_info): sname = software_info['sname'] @@ -306,7 +308,7 @@ chmod +x {install_script} result = self.exe.exec_raw(install_cmd) if result: print(f"install to {install_path} successful") - self.set_installed_status(install_path) + self.set_installed_status(install_path, "1") else: print("install failed") sys.exit() @@ -354,4 +356,30 @@ chmod +x {depend_file} ./{depend_file} ''' self.exe.exec_raw(run_cmd) - + + def remove(self, software_info): + self.tool.prt_content("UNINSTALL " + software_info) + file_list = [d for d in glob(self.SOFTWARE_PATH+'/**', recursive=True)] + remove_list = [] + for file in file_list: + if software_info in file and os.path.isdir(file) and self.is_installed(file): + remove_list.append(file) + lens = len(remove_list) + if lens == 0: + print("software not installed") + return + choice = 1 + if lens > 1: + for i in range(lens): + print(f"{i+1}: {remove_list[i]}") + self.tool.prt_content("") + choice = input(f"find {lens} software, which one do you want to remove?\n") + choice = int(choice) + if choice > lens or choice < 1: + print("Wrong choice!") + return + self.set_installed_status(remove_list[choice-1], "0") + print("Successfully remove "+software_info) + + + diff --git a/src/jarvis.py b/src/jarvis.py index 3c5900c5c73a33bc3b2a608d923ab1ce0a430a31..eab4fb49b2d0822507137c4c1fe7a367f7e8021d 100644 --- a/src/jarvis.py +++ b/src/jarvis.py @@ -16,6 +16,8 @@ class Jarvis: parser.add_argument("-i","--info", help=f"get machine info", action="store_true") #accept software_name/version GCC/GCC+MPI/CLANG/CLANG+MPI parser.add_argument("-install","--install", help=f"install dependency", nargs=2) + #accept software_name/version GCC/GCC+MPI/CLANG/CLANG+MPI + parser.add_argument("-remove","--remove", help=f"remove software", nargs=1) # dependency install parser.add_argument("-dp","--depend", help=f"{DataService.app_name} dependency install", action="store_true") parser.add_argument("-e","--env", help=f"set environment {DataService.app_name}", action="store_true") @@ -54,7 +56,10 @@ class Jarvis: if self.args.install: self.analysis.install(self.args.install[0], self.args.install[1]) - + + if self.args.remove: + self.analysis.remove(self.args.remove[0]) + if self.args.env: self.analysis.env()