From c3bafd8765fccd8117b6d66e3cc7214af3a759e9 Mon Sep 17 00:00:00 2001 From: iotwins Date: Fri, 1 Apr 2022 16:11:29 +0800 Subject: [PATCH 1/2] Add container def file autogenerate function --- package/go/1.18/install.sh | 7 ++-- package/singularity/3.9.6/install.sh | 14 ++++---- src/analysisService.py | 5 +++ src/containerService.py | 50 ++++++++++++++++++++++++++++ src/dataService.py | 8 ++--- src/jarvis.py | 5 +++ 6 files changed, 76 insertions(+), 13 deletions(-) mode change 100644 => 100755 package/singularity/3.9.6/install.sh create mode 100644 src/containerService.py diff --git a/package/go/1.18/install.sh b/package/go/1.18/install.sh index fa986d2..2c13298 100755 --- a/package/go/1.18/install.sh +++ b/package/go/1.18/install.sh @@ -3,9 +3,10 @@ set -x set -e cd ${JARVIS_TMP} -if [ ! -f "go1.18.linux-arm64.tar.gz" ]; then -wget https://go.dev/dl/go1.18.linux-arm64.tar.gz +file_name='go1.18.linux-arm64.tar.gz' +if [ ! -f "${JARVIS_DOWNLOAD}/${file_name}" ]; then +wget --no-check-certificate -P ${JARVIS_DOWNLOAD} https://go.dev/dl/${file_name} fi rm -rf /usr/local/go -tar -xzvf go1.18.linux-arm64.tar.gz -C /usr/local +tar -xzvf ${JARVIS_DOWNLOAD}/${file_name} -C /usr/local export PATH=/usr/local/go/bin:$PATH diff --git a/package/singularity/3.9.6/install.sh b/package/singularity/3.9.6/install.sh old mode 100644 new mode 100755 index bb51f01..bdaa619 --- a/package/singularity/3.9.6/install.sh +++ b/package/singularity/3.9.6/install.sh @@ -3,13 +3,15 @@ set -x set -e cd ${JARVIS_TMP} export PATH=/usr/local/go/bin:$PATH -sudo yum install -y libseccomp-devel squashfs-tools cryptsetup -if [ ! -f "singularity-ce-3.9.6.tar.gz" ]; then -wget https://github.com/sylabs/singularity/releases/download/v3.9.6/singularity-ce-3.9.6.tar.gz +#sudo yum install -y libseccomp-devel squashfs-tools cryptsetup +file_noext='singularity-ce-3.9.6' +file_name="${file_noext}.tar.gz" +if [ ! -f "${JARVIS_DOWNLOAD}/${file_name}" ]; then + wget --no-check-certificate -P ${JARVIS_DOWNLOAD} https://github.com/sylabs/singularity/releases/download/v3.9.6/${file_name} fi -rm -rf singularity-ce-3.9.6 -tar -xzvf singularity-ce-3.9.6.tar.gz -cd singularity-ce-3.9.6 +rm -rf ${file_noext} +tar -xzvf ${JARVIS_DOWNLOAD}/${file_name} +cd ${file_noext} ./mconfig make -C builddir sudo make -C builddir install \ No newline at end of file diff --git a/src/analysisService.py b/src/analysisService.py index 758c44d..d6796ed 100644 --- a/src/analysisService.py +++ b/src/analysisService.py @@ -11,6 +11,7 @@ from runService import RunService from perfService import PerfService from testService import TestService from benchService import BenchmarkService +from containerService import ContainerService class AnalysisService: def __init__(self): @@ -23,6 +24,7 @@ class AnalysisService: self.jrun = RunService() self.jperf = PerfService() self.jbenchmark = BenchmarkService() + self.jcontainer = ContainerService() self.jtest = TestService() def get_machine_info(self): @@ -75,3 +77,6 @@ class AnalysisService: def install_deps(self): self.jinstall.install_depend() + + def gen_def(self, image): + self.jcontainer.gen_def(image) diff --git a/src/containerService.py b/src/containerService.py new file mode 100644 index 0000000..348a1e5 --- /dev/null +++ b/src/containerService.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from toolService import ToolService +from dataService import DataService + +class ContainerService: + def __init__(self): + self.tool = ToolService() + self.data = DataService() + + def gen_def(self, image): + config_file = self.data.get_config_file_name() + def_file_content = f'''BootStrap: docker +From: {image} + +%environment + source /etc/profile || true + cd /hpcrunner + source env.sh + +%post + # Install the necessary development environment + yum install -y environment-modules git dmidecode pciutils wget vim + # Install base gcc + yum install -y gcc gcc-c++ gcc-gfortran glibc-devel make libgfortran + # install network package + yum install -y tcsh tcl lsof tk bc + source /etc/profile || true + git config --global http.sslVerify false + git clone https://gitee.com/openeuler/hpcrunner + cd hpcrunner + source ./init.sh + # add your personal files here + + # Switch config + ./jarvis -use {config_file} + # download dependency + ./jarvis -d + # install dependency + ./jarvis -dp + # build hpc + ./jarvis -b + # run hpc + ./jarvis -r + # clean tmp directory + rm -rf downloads tmp +''' + def_file = config_file+'.def' + self.tool.write_file(def_file, def_file_content) + print(f'Singularity file {def_file} successfully generated!') diff --git a/src/dataService.py b/src/dataService.py index e4e481a..99a202f 100644 --- a/src/dataService.py +++ b/src/dataService.py @@ -25,6 +25,7 @@ class DataService: run_cmd = {} batch_cmd = '' #Other Info + config_file = 'data.config' meta_file = '.meta' root_path = os.getcwd() download_info = '' @@ -41,14 +42,13 @@ class DataService: self.tool = ToolService() self.data_process() - def get_file_name(self): - file_name = 'data.config' + def get_config_file_name(self): if not os.path.exists(DataService.meta_file): - return file_name + return DataService.config_file return self.tool.read_file(DataService.meta_file) def get_data_config(self): - file_name = self.get_file_name() + file_name = self.get_config_file_name() file_path = self.get_abspath(file_name) with open(file_path, encoding='utf-8') as file_obj: contents = file_obj.read() diff --git a/src/jarvis.py b/src/jarvis.py index 01d744e..3c5900c 100644 --- a/src/jarvis.py +++ b/src/jarvis.py @@ -34,6 +34,8 @@ class Jarvis: parser.add_argument("-rb","--rbatch", help=f"run batch {DataService.app_name}", action="store_true") # batch download parser.add_argument("-d","--download", help="Batch Download...", action="store_true") + # generate singularity def file + parser.add_argument("-container","--container", help="generate container file...", nargs=1) parser.add_argument("-net","--network", help="network checking...", action="store_true") #change yum repo to aliyun parser.add_argument("-yum","--yum", help="yum repo changing...", action="store_true") @@ -86,6 +88,9 @@ class Jarvis: if self.args.ncuperf: self.analysis.ncu_perf(self.args.ncuperf[0]) + if self.args.container: + self.analysis.gen_def(self.args.container[0]) + if self.args.use: self.analysis.switch_config(self.args.use[0]) -- Gitee From 042439496def9687838fedf07f44edfe3e738b89 Mon Sep 17 00:00:00 2001 From: iotwins Date: Fri, 1 Apr 2022 16:29:06 +0800 Subject: [PATCH 2/2] fix git ignore issue --- .gitignore | 64 ++++++++++++++++++++++++++++++++++++---- src/configService.py | 5 ++-- src/perfService.py | 69 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 src/perfService.py diff --git a/.gitignore b/.gitignore index 3d2d3e5..6211baa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,65 @@ -perf* +# Compiled source # +################### +*.a +*.com +*.class +*.dll +*.exe +*.o +*.o.d +*.py[ocd] +*.so + +# Packages # +############ +# it's better to unpack these files and commit the raw source +# git has its own built in compression methods +*.7z +*.bz2 +*.bzip2 +*.dmg +*.gz +*.iso +*.jar +*.rar +*.tar +*.tbz2 +*.tgz +*.zip + +# Python files # +################ +# setup.py working directory +build +# sphinx build directory +_build +# setup.py dist directory +dist +doc/build +doc/cdoc/build +# Egg metadata +*.egg-info +# The shelf plugin uses this dir +./.shelf +MANIFEST +.cache +pip-wheel-metadata +.python-version + +# Logs and databases # +###################### +*.log +*.sql +*.sqlite +# Things specific to this project # +###################### +perf.data.* kperf.data.* env.sh hostfile .vscode -test.* -porting* -HPC-info* tmp downloads/* -runner.log depend_install.sh -.meta \ No newline at end of file +.meta +software/* \ No newline at end of file diff --git a/src/configService.py b/src/configService.py index ba1972f..ae0652a 100644 --- a/src/configService.py +++ b/src/configService.py @@ -10,9 +10,10 @@ class ConfigService: self.exe = ExecuteService() self.tool = ToolService() self.ROOT = os.getcwd() + self.meta_path = os.path.join(self.ROOT, DataService.meta_file) def switch_config(self, config_file): print(f"Switch config file to {config_file}") - meta_path = os.path.join(self.ROOT, DataService.meta_file) - self.tool.write_file(meta_path, config_file.strip()) + self.tool.write_file(self.meta_path, config_file.strip()) print("Successfully switched.") + diff --git a/src/perfService.py b/src/perfService.py new file mode 100644 index 0000000..ee4c2d7 --- /dev/null +++ b/src/perfService.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import platform +import sys +import re + +from dataService import DataService +from executeService import ExecuteService +from toolService import ToolService + +class PerfService: + def __init__(self): + self.hpc_data = DataService() + self.exe = ExecuteService() + self.tool = ToolService() + self.isARM = platform.machine() == 'aarch64' + + def get_pid(self): + #get pid + pid_cmd = f'pidof {DataService.binary_file}' + result = self.exe.exec_popen(pid_cmd) + if len(result) == 0: + print("failed to get pid.") + sys.exit() + else: + pid_list = result[0].split(' ') + mid = int(len(pid_list)/2) + return pid_list[mid].strip() + + def perf(self): + print(f"start perf {DataService.app_name}") + #get pid + pid = self.get_pid() + #start perf && analysis + perf_cmd = f''' +perf record {DataService.perf_para} -a -g -p {pid} +perf report -i ./perf.data -F period,sample,overhead,symbol,dso,comm -s overhead --percent-limit 0.1% --stdio +''' + self.exe.exec_raw(perf_cmd) + + def get_arch(self): + arch = 'arm' + if not self.isARM: + arch = 'X86' + return arch + + def get_cur_time(self): + return re.sub(' |:', '-', self.tool.get_time_stamp()) + + def gpu_perf(self): + print(f"start gpu perf") + run_cmd = self.hpc_data.get_run() + gperf_cmd = f''' +{self.hpc_data.get_env()} +cd {DataService.case_dir} +nsys profile -y 5s -d 100s {DataService.nsys_para} -o nsys-{self.get_arch()}-{self.get_cur_time()} {run_cmd} + ''' + self.exe.exec_raw(gperf_cmd) + + def ncu_perf(self, kernel): + print(f"start ncu perf") + run_cmd = self.hpc_data.get_run() + ncu_cmd = f''' +{self.hpc_data.get_env()} +cd {DataService.case_dir} +ncu --export ncu-{self.get_arch()}-{self.get_cur_time()} {DataService.ncu_para} --import-source=yes --set full --kernel-name {kernel} --launch-skip 1735 --launch-count 1 {run_cmd} +''' + self.exe.exec_raw(ncu_cmd) + -- Gitee