diff --git a/src/analysisService.py b/src/analysisService.py new file mode 100644 index 0000000000000000000000000000000000000000..758c44dc904b22c021f5eaaa05442aa2f3d167db --- /dev/null +++ b/src/analysisService.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from machineService import MachineService +from configService import ConfigService +from downloadService import DownloadService +from installService import InstallService +from envService import EnvService +from buildService import BuildService +from runService import RunService +from perfService import PerfService +from testService import TestService +from benchService import BenchmarkService + +class AnalysisService: + def __init__(self): + self.jmachine = MachineService() + self.jconfig = ConfigService() + self.jdownload = DownloadService() + self.jinstall = InstallService() + self.jenv = EnvService() + self.jbuild = BuildService() + self.jrun = RunService() + self.jperf = PerfService() + self.jbenchmark = BenchmarkService() + self.jtest = TestService() + + def get_machine_info(self): + self.jmachine.output_machine_info() + + def bench(self, bench_case): + self.jbenchmark.output_bench_info(bench_case) + + def switch_config(self, config_file): + self.jconfig.switch_config(config_file) + + def test(self): + self.jtest.test() + + def download(self): + self.jdownload.download() + + def check_network(self): + self.jdownload.check_network() + + def gpu_perf(self): + self.jperf.gpu_perf() + + def ncu_perf(self, kernel): + self.jperf.ncu_perf(kernel) + + def perf(self): + self.jperf.perf() + + def kperf(self): + self.jperf.kperf() + + def run(self): + self.jrun.run() + + def batch_run(self): + self.jrun.batch_run() + + def clean(self): + self.jbuild.clean() + + def build(self): + self.jbuild.build() + + def env(self): + self.jenv.env() + + def install(self,software_path, compiler_mpi_info): + self.jinstall.install(software_path, compiler_mpi_info) + + def install_deps(self): + self.jinstall.install_depend() diff --git a/src/bench.py b/src/benchService.py similarity index 89% rename from src/bench.py rename to src/benchService.py index 96f55d70c9ac7b4912c28041dd66b919b57b132e..28a0d8509ee8f15b0c6a4a5c04f0f1be4a658d13 100644 --- a/src/bench.py +++ b/src/benchService.py @@ -4,13 +4,13 @@ import platform import os from glob import glob -from execute import Execute +from executeService import ExecuteService -class Benchmark: +class BenchmarkService: def __init__(self): self.isARM = platform.machine() == 'aarch64' self.ROOT = os.getcwd() - self.exe = Execute() + self.exe = ExecuteService() self.RUN_FILE = 'run.sh' self.ALL = 'all' diff --git a/src/buildService.py b/src/buildService.py new file mode 100644 index 0000000000000000000000000000000000000000..f82580283b5c490725e3e9f43ceb60f5e79f98f9 --- /dev/null +++ b/src/buildService.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from dataService import DataService +from executeService import ExecuteService + +class BuildService: + def __init__(self): + self.hpc_data = DataService() + self.exe = ExecuteService() + + def clean(self): + print(f"start clean {DataService.app_name}") + clean_cmd=self.hpc_data.get_clean_cmd() + self.exe.exec_raw(clean_cmd) + + def build(self): + print(f"start build {DataService.app_name}") + build_cmd = self.hpc_data.get_build_cmd() + self.exe.exec_raw(build_cmd) diff --git a/src/configService.py b/src/configService.py new file mode 100644 index 0000000000000000000000000000000000000000..ba1972fb0bb54abf6733b834411269e10d59c136 --- /dev/null +++ b/src/configService.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +from dataService import DataService +from executeService import ExecuteService +from toolService import ToolService + +class ConfigService: + def __init__(self): + self.exe = ExecuteService() + self.tool = ToolService() + self.ROOT = os.getcwd() + + def switch_config(self, config_file): + print(f"Switch config file to {config_file}") + meta_path = os.path.join(self.ROOT, DataService.meta_file) + self.tool.write_file(meta_path, config_file.strip()) + print("Successfully switched.") diff --git a/src/data.py b/src/dataService.py similarity index 66% rename from src/data.py rename to src/dataService.py index 31bb4d2bd2b3a73e997613057648e35031339528..e4e481ae353bcce9672983d2ca4b561cacbab14d 100644 --- a/src/data.py +++ b/src/dataService.py @@ -3,9 +3,9 @@ import os import platform -from tool import Tool +from toolService import ToolService -class Data: +class DataService: # Hardware Info avail_ips='' # Dependent Software environment Info @@ -34,18 +34,18 @@ class Data: nsys_para = '' ncu_para = '' def get_abspath(self, relpath): - return os.path.join(Data.root_path, relpath) + return os.path.join(DataService.root_path, relpath) def __init__(self): self.isARM = platform.machine() == 'aarch64' - self.tool = Tool() + self.tool = ToolService() self.data_process() def get_file_name(self): file_name = 'data.config' - if not os.path.exists(Data.meta_file): + if not os.path.exists(DataService.meta_file): return file_name - return self.tool.read_file(Data.meta_file) + return self.tool.read_file(DataService.meta_file) def get_data_config(self): file_name = self.get_file_name() @@ -83,16 +83,16 @@ class Data: return start_row, data def set_app_info(self, data): - Data.app_name = data['app_name'] - Data.build_dir = data['build_dir'] - Data.binary_dir = data['binary_dir'] - Data.case_dir = data['case_dir'] + DataService.app_name = data['app_name'] + DataService.build_dir = data['build_dir'] + DataService.binary_dir = data['binary_dir'] + DataService.case_dir = data['case_dir'] def set_perf_info(self, data): - Data.kperf_para = data['kperf'] - Data.perf_para = data['perf'] - Data.nsys_para = data['nsys'] - Data.ncu_para = data['ncu'] + DataService.kperf_para = data['kperf'] + DataService.perf_para = data['perf'] + DataService.nsys_para = data['nsys'] + DataService.ncu_para = data['ncu'] def split_two_part(self, data): split_list = data.split(' ', 1) @@ -111,62 +111,62 @@ class Data: while rowIndex < len(rows): row = rows[rowIndex].strip() if row == '[SERVER]': - rowIndex, Data.avail_ips = self.read_rows(rows, rowIndex+1) + rowIndex, DataService.avail_ips = self.read_rows(rows, rowIndex+1) elif row == '[DOWNLOAD]': - rowIndex, Data.download_info = self.read_rows(rows, rowIndex+1) + rowIndex, DataService.download_info = self.read_rows(rows, rowIndex+1) elif row == '[DEPENDENCY]': - rowIndex, Data.dependency = self.read_rows(rows, rowIndex+1) + rowIndex, DataService.dependency = self.read_rows(rows, rowIndex+1) elif row == '[ENV]': - rowIndex, Data.module_content = self.read_rows(rows, rowIndex+1) + rowIndex, DataService.module_content = self.read_rows(rows, rowIndex+1) elif row == '[APP]': rowIndex, data = self.read_rows_kv(rows, rowIndex+1) self.set_app_info(data) elif row == '[BUILD]': - rowIndex, Data.build_cmd = self.read_rows(rows, rowIndex+1) + rowIndex, DataService.build_cmd = self.read_rows(rows, rowIndex+1) elif row == '[CLEAN]': - rowIndex, Data.clean_cmd = self.read_rows(rows, rowIndex+1) + rowIndex, DataService.clean_cmd = self.read_rows(rows, rowIndex+1) elif row == '[RUN]': - rowIndex, Data.run_cmd = self.read_rows_kv(rows, rowIndex+1) + rowIndex, DataService.run_cmd = self.read_rows_kv(rows, rowIndex+1) elif row == '[BATCH]': - rowIndex, Data.batch_cmd = self.read_rows(rows, rowIndex+1) + rowIndex, DataService.batch_cmd = self.read_rows(rows, rowIndex+1) elif row == '[PERF]': rowIndex, perf_data = self.read_rows_kv(rows, rowIndex+1) self.set_perf_info(perf_data) else: rowIndex += 1 - Data.binary_file, Data.binary_para = self.split_two_part(Data.run_cmd['binary']) + DataService.binary_file, DataService.binary_para = self.split_two_part(DataService.run_cmd['binary']) def get_clean_cmd(self): return f''' -cd {Data.build_dir} -{Data.clean_cmd} +cd {DataService.build_dir} +{DataService.clean_cmd} ''' def get_env(self): return f''' ./jarvis -e -source ./{Data.env_file}''' +source ./{DataService.env_file}''' def get_build_cmd(self): return f''' {self.get_env()} -cd {Data.build_dir} -{Data.build_cmd} +cd {DataService.build_dir} +{DataService.build_cmd} ''' def get_run(self): - nodes = int(Data.run_cmd['nodes']) - run_cmd = Data.run_cmd['run'] + nodes = int(DataService.run_cmd['nodes']) + run_cmd = DataService.run_cmd['run'] hostfile = '' if nodes > 1: hostfile = '--hostfile hostfile' if 'mpi' in run_cmd: run_cmd = f'{run_cmd} {hostfile}' - binary = os.path.join(Data.binary_dir, Data.binary_file) - return f'''{run_cmd} {binary} {Data.binary_para}''' + binary = os.path.join(DataService.binary_dir, DataService.binary_file) + return f'''{run_cmd} {binary} {DataService.binary_para}''' def get_run_cmd(self): return f''' {self.get_env()} -cd {Data.case_dir} +cd {DataService.case_dir} {self.get_run()} ''' \ No newline at end of file diff --git a/src/downloadService.py b/src/downloadService.py new file mode 100644 index 0000000000000000000000000000000000000000..4a644a76edc097cd766e44553d8915eb33d41fc6 --- /dev/null +++ b/src/downloadService.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +from dataService import DataService +from executeService import ExecuteService +from toolService import ToolService + +class DownloadService: + def __init__(self): + self.hpc_data = DataService() + self.exe = ExecuteService() + self.tool = ToolService() + self.ROOT = os.getcwd() + self.download_list = self.tool.gen_list(DataService.download_info) + self.download_path = os.path.join(self.ROOT, 'downloads') + self.package_path = os.path.join(self.ROOT, 'package') + + def check_network(self): + print(f"start network checking") + network_test_cmd=''' +wget --spider -T 5 -q -t 2 www.baidu.com | echo $? +curl -s -o /dev/null www.baidu.com | echo $? + ''' + self.exe.exec_raw(network_test_cmd) + + def change_yum_repo(self): + print(f"start yum repo change") + repo_cmd = ''' +cp ./templates/yum/*.repo /etc/yum.repos.d/ +yum clean all +yum makecache +''' + self.exe.exec_raw(repo_cmd) + + def gen_wget_url(self, out_dir='./downloads', url='', filename=''): + head = "wget --no-check-certificate" + file_path = os.path.join(out_dir, filename) + download_url = f'{head} {url} -O {file_path}' + return download_url + + def download(self): + print(f"start download") + filename_url_map = {} + self.tool.mkdirs(self.download_path) + download_flag = False + # create directory + for url_info in self.download_list: + url_list = url_info.split(' ') + if len(url_list) < 2: + continue + software_info = url_list[0].strip() + url_link = url_list[1].strip() + filename = os.path.basename(url_link) + if len(url_list) == 3: + filename = url_list[2].strip() + filename_url_map[filename] = url_link + # create software directory + software_path = os.path.join(self.package_path, software_info) + self.tool.mkdirs(software_path) + # create install script + install_script = os.path.join(software_path, "install.sh") + self.tool.mkfile(install_script) + print(filename_url_map) + # start download + for filename, url in filename_url_map.items(): + download_flag = True + file_path = os.path.join(self.download_path, filename) + if os.path.exists(file_path): + self.tool.prt_content(f"FILE {filename} already DOWNLOADED") + continue + download_url = self.gen_wget_url(self.download_path, url, filename) + self.tool.prt_content("DOWNLOAD " + filename) + output = os.popen(download_url) + data = output.read() + output.close() + + if not download_flag: + print("The download list is empty!") diff --git a/src/envService.py b/src/envService.py new file mode 100644 index 0000000000000000000000000000000000000000..fbdfde030de7e766a1ebfeab79b64cc4cba6a59b --- /dev/null +++ b/src/envService.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os + +from dataService import DataService +from toolService import ToolService +from executeService import ExecuteService + +class EnvService: + def __init__(self): + self.hpc_data = DataService() + self.tool = ToolService() + self.ROOT = os.getcwd() + self.exe = ExecuteService() + + def env(self): + print(f"set environment {DataService.app_name}") + env_file = os.path.join(self.ROOT, DataService.env_file) + self.tool.write_file(env_file, DataService.module_content) + print(f"ENV FILE {DataService.env_file} GENERATED.") + self.exe.exec_raw(f'chmod +x {DataService.env_file}') diff --git a/src/execute.py b/src/executeService.py similarity index 95% rename from src/execute.py rename to src/executeService.py index 19e6b50f283d2bfb08a61d6fc1946e8d5782162b..b2dfe1f6b54f41038c10b2478fa50e458e29cc93 100644 --- a/src/execute.py +++ b/src/executeService.py @@ -4,17 +4,17 @@ import os import logging from asyncio.log import logger from datetime import datetime -from tool import Tool +from toolService import ToolService LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s" DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p" logging.basicConfig(filename='runner.log', level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) -class Execute: +class ExecuteService: def __init__(self): self.cur_time = '' self.end_time = '' - self.tool = Tool() + self.tool = ToolService() self.flags = '*' * 80 self.end_flag = 'END: ' diff --git a/src/analysis.py b/src/installService.py similarity index 60% rename from src/analysis.py rename to src/installService.py index fad7b530a3d247bc98e26b97d4435443e690005f..62c115fc37d071a7a9050f8fba05a32915f75d10 100644 --- a/src/analysis.py +++ b/src/installService.py @@ -1,30 +1,26 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import platform -import sys import os +import sys import re +from enum import Enum from glob import glob -from data import Data -from tool import Tool -from execute import Execute -from machine import Machine -from bench import Benchmark +from dataService import DataService +from toolService import ToolService +from executeService import ExecuteService -from enum import Enum - class SType(Enum): COMPILER = 1 MPI = 2 UTIL = 3 LIB = 4 -class Install: +class InstallService: def __init__(self): - self.hpc_data = Data() - self.exe = Execute() - self.tool = Tool() + self.hpc_data = DataService() + self.exe = ExecuteService() + self.tool = ToolService() self.ROOT = os.getcwd() self.PACKAGE_PATH = os.path.join(self.ROOT, 'package') self.COMPILER_PATH = os.path.join(self.ROOT, 'software/compiler') @@ -348,9 +344,9 @@ chmod +x {install_script} def install_depend(self): depend_file = 'depend_install.sh' - print(f"start installing dependendcy of {Data.app_name}") + print(f"start installing dependendcy of {DataService.app_name}") depend_content = f''' -{Data.dependency} +{DataService.dependency} ''' self.tool.write_file(depend_file, depend_content) run_cmd = f''' @@ -359,294 +355,3 @@ chmod +x {depend_file} ''' self.exe.exec_raw(run_cmd) -class Env: - def __init__(self): - self.hpc_data = Data() - self.tool = Tool() - self.ROOT = os.getcwd() - self.exe = Execute() - - def env(self): - print(f"set environment {Data.app_name}") - env_file = os.path.join(self.ROOT, Data.env_file) - self.tool.write_file(env_file, Data.module_content) - print(f"ENV FILE {Data.env_file} GENERATED.") - self.exe.exec_raw(f'chmod +x {Data.env_file}') - -class Build: - def __init__(self): - self.hpc_data = Data() - self.exe = Execute() - - def clean(self): - print(f"start clean {Data.app_name}") - clean_cmd=self.hpc_data.get_clean_cmd() - self.exe.exec_raw(clean_cmd) - - def build(self): - print(f"start build {Data.app_name}") - build_cmd = self.hpc_data.get_build_cmd() - self.exe.exec_raw(build_cmd) - -class Run: - def __init__(self): - self.hpc_data = Data() - self.exe = Execute() - self.tool = Tool() - self.ROOT = os.getcwd() - self.avail_ips_list = self.tool.gen_list(Data.avail_ips) - - def gen_hostfile(self, nodes): - length = len(self.avail_ips_list) - if nodes > length: - print(f"You don't have {nodes} nodes, only {length} nodes available!") - sys.exit() - if nodes <= 1: - return - gen_nodes = '\n'.join(self.avail_ips_list[:nodes]) - print(f"HOSTFILE\n{gen_nodes}\nGENERATED.") - self.tool.write_file('hostfile', gen_nodes) - - # single run - def run(self): - print(f"start run {Data.app_name}") - nodes = int(Data.run_cmd['nodes']) - self.gen_hostfile(nodes) - run_cmd = self.hpc_data.get_run_cmd() - self.exe.exec_raw(run_cmd) - - def batch_run(self): - batch_file = 'batch_run.sh' - batch_file_path = os.path.join(self.ROOT, batch_file) - print(f"start batch run {Data.app_name}") - batch_content = f''' -cd {Data.case_dir} -{Data.batch_cmd} -''' - self.tool.write_file(batch_file_path, batch_content) - run_cmd = f''' -chmod +x {batch_file} -./{batch_file} -''' - self.exe.exec_raw(run_cmd) - -class Perf: - def __init__(self): - self.hpc_data = Data() - self.exe = Execute() - self.tool = Tool() - self.isARM = platform.machine() == 'aarch64' - - def get_pid(self): - #get pid - pid_cmd = f'pidof {Data.binary_file}' - result = self.exe.exec_popen(pid_cmd) - if len(result) == 0: - print("failed to get pid.") - sys.exit() - else: - pid_list = result[0].split(' ') - mid = int(len(pid_list)/2) - return pid_list[mid].strip() - - def perf(self): - print(f"start perf {Data.app_name}") - #get pid - pid = self.get_pid() - #start perf && analysis - perf_cmd = f''' -perf record {Data.perf_para} -a -g -p {pid} -perf report -i ./perf.data -F period,sample,overhead,symbol,dso,comm -s overhead --percent-limit 0.1% --stdio -''' - self.exe.exec_raw(perf_cmd) - - def get_arch(self): - arch = 'arm' - if not self.isARM: - arch = 'X86' - return arch - - def get_cur_time(self): - return re.sub(' |:', '-', self.tool.get_time_stamp()) - - def gpu_perf(self): - print(f"start gpu perf") - run_cmd = self.hpc_data.get_run() - gperf_cmd = f''' -{self.hpc_data.get_env()} -cd {Data.case_dir} -nsys profile -y 5s -d 100s {Data.nsys_para} -o nsys-{self.get_arch()}-{self.get_cur_time()} {run_cmd} - ''' - self.exe.exec_raw(gperf_cmd) - - def ncu_perf(self, kernel): - print(f"start ncu perf") - run_cmd = self.hpc_data.get_run() - ncu_cmd = f''' -{self.hpc_data.get_env()} -cd {Data.case_dir} -ncu --export ncu-{self.get_arch()}-{self.get_cur_time()} {Data.ncu_para} --import-source=yes --set full --kernel-name {kernel} --launch-skip 1735 --launch-count 1 {run_cmd} -''' - self.exe.exec_raw(ncu_cmd) - -class Download: - def __init__(self): - self.hpc_data = Data() - self.exe = Execute() - self.tool = Tool() - self.ROOT = os.getcwd() - self.download_list = self.tool.gen_list(Data.download_info) - self.download_path = os.path.join(self.ROOT, 'downloads') - self.package_path = os.path.join(self.ROOT, 'package') - - def check_network(self): - print(f"start network checking") - network_test_cmd=''' -wget --spider -T 5 -q -t 2 www.baidu.com | echo $? -curl -s -o /dev/null www.baidu.com | echo $? - ''' - self.exe.exec_raw(network_test_cmd) - - def change_yum_repo(self): - print(f"start yum repo change") - repo_cmd = ''' -cp ./templates/yum/*.repo /etc/yum.repos.d/ -yum clean all -yum makecache -''' - self.exe.exec_raw(repo_cmd) - - def gen_wget_url(self, out_dir='./downloads', url='', filename=''): - head = "wget --no-check-certificate" - file_path = os.path.join(out_dir, filename) - download_url = f'{head} {url} -O {file_path}' - return download_url - - def download(self): - print(f"start download") - filename_url_map = {} - self.tool.mkdirs(self.download_path) - download_flag = False - # create directory - for url_info in self.download_list: - url_list = url_info.split(' ') - if len(url_list) < 2: - continue - software_info = url_list[0].strip() - url_link = url_list[1].strip() - filename = os.path.basename(url_link) - if len(url_list) == 3: - filename = url_list[2].strip() - filename_url_map[filename] = url_link - # create software directory - software_path = os.path.join(self.package_path, software_info) - self.tool.mkdirs(software_path) - # create install script - install_script = os.path.join(software_path, "install.sh") - self.tool.mkfile(install_script) - print(filename_url_map) - # start download - for filename, url in filename_url_map.items(): - download_flag = True - file_path = os.path.join(self.download_path, filename) - if os.path.exists(file_path): - self.tool.prt_content(f"FILE {filename} already DOWNLOADED") - continue - download_url = self.gen_wget_url(self.download_path, url, filename) - self.tool.prt_content("DOWNLOAD " + filename) - output = os.popen(download_url) - data = output.read() - output.close() - - if not download_flag: - print("The download list is empty!") -class Test: - def __init__(self): - self.exe = Execute() - self.ROOT = os.getcwd() - self.test_dir = os.path.join(self.ROOT, 'test') - - def test(self): - run_cmd = f''' -cd {self.test_dir} -./test-qe.sh -cd {self.test_dir} -./test-util.sh -''' - self.exe.exec_raw(run_cmd) - -class Config: - def __init__(self): - self.exe = Execute() - self.tool = Tool() - self.ROOT = os.getcwd() - - def switch_config(self, config_file): - print(f"Switch config file to {config_file}") - meta_path = os.path.join(self.ROOT, Data.meta_file) - self.tool.write_file(meta_path, config_file.strip()) - print("Successfully switched.") - -class Analysis: - def __init__(self): - self.jmachine = Machine() - self.jtest = Test() - self.jdownload = Download() - self.jbenchmark = Benchmark() - self.jperf = Perf() - self.jrun = Run() - self.jbuild = Build() - self.jenv = Env() - self.jinstall = Install() - self.jconfig = Config() - - def get_machine_info(self): - self.jmachine.output_machine_info() - - def bench(self, bench_case): - self.jbenchmark.output_bench_info(bench_case) - - def switch_config(self, config_file): - self.jconfig.switch_config(config_file) - - def test(self): - self.jtest.test() - - def download(self): - self.jdownload.download() - - def check_network(self): - self.jdownload.check_network() - - def gpu_perf(self): - self.jperf.gpu_perf() - - def ncu_perf(self, kernel): - self.jperf.ncu_perf(kernel) - - def perf(self): - self.jperf.perf() - - def kperf(self): - self.jperf.kperf() - - def run(self): - self.jrun.run() - - def batch_run(self): - self.jrun.batch_run() - - def clean(self): - self.jbuild.clean() - - def build(self): - self.jbuild.build() - - def env(self): - self.jenv.env() - - def install(self,software_path, compiler_mpi_info): - self.jinstall.install(software_path, compiler_mpi_info) - - def install_deps(self): - self.jinstall.install_depend() diff --git a/src/jarvis.py b/src/jarvis.py index 5b03d64df3d1a3d54541cec285c74609fe490fbe..01d744e5333cf502b51654ea1992cfcf66613274 100644 --- a/src/jarvis.py +++ b/src/jarvis.py @@ -2,14 +2,14 @@ # -*- coding: utf-8 -*- import argparse -from data import Data -from analysis import Analysis +from dataService import DataService +from analysisService import AnalysisService class Jarvis: def __init__(self): - self.analysis = Analysis() + self.analysis = AnalysisService() # Argparser set - parser = argparse.ArgumentParser(description=f'please put me into CASE directory, used for {Data.app_name} Compiler/Clean/Run/Compare', + parser = argparse.ArgumentParser(description=f'please put me into CASE directory, used for {DataService.app_name} Compiler/Clean/Run/Compare', usage='%(prog)s [-h] [--build] [--clean] [...]') parser.add_argument("-v","--version", help=f"get version info", action="store_true") parser.add_argument("-use","--use", help="Switch config file...", nargs=1) @@ -17,21 +17,21 @@ class Jarvis: #accept software_name/version GCC/GCC+MPI/CLANG/CLANG+MPI parser.add_argument("-install","--install", help=f"install dependency", nargs=2) # dependency install - parser.add_argument("-dp","--depend", help=f"{Data.app_name} dependency install", action="store_true") - parser.add_argument("-e","--env", help=f"set environment {Data.app_name}", action="store_true") - parser.add_argument("-b","--build", help=f"compile {Data.app_name}", action="store_true") - parser.add_argument("-cls","--clean", help=f"clean {Data.app_name}", action="store_true") - parser.add_argument("-r","--run", help=f"run {Data.app_name}", action="store_true") - parser.add_argument("-p","--perf", help=f"auto perf {Data.app_name}", action="store_true") - parser.add_argument("-kp","--kperf", help=f"auto kperf {Data.app_name}", action="store_true") + parser.add_argument("-dp","--depend", help=f"{DataService.app_name} dependency install", action="store_true") + parser.add_argument("-e","--env", help=f"set environment {DataService.app_name}", action="store_true") + parser.add_argument("-b","--build", help=f"compile {DataService.app_name}", action="store_true") + parser.add_argument("-cls","--clean", help=f"clean {DataService.app_name}", action="store_true") + parser.add_argument("-r","--run", help=f"run {DataService.app_name}", action="store_true") + parser.add_argument("-p","--perf", help=f"auto perf {DataService.app_name}", action="store_true") + parser.add_argument("-kp","--kperf", help=f"auto kperf {DataService.app_name}", action="store_true") # GPU perf parser.add_argument("-gp","--gpuperf", help="GPU perf...", action="store_true") # NCU perf parser.add_argument("-ncu","--ncuperf", help="NCU perf...", nargs=1) - parser.add_argument("-c","--compare", help=f"compare {Data.app_name}", nargs=2) + parser.add_argument("-c","--compare", help=f"compare {DataService.app_name}", nargs=2) # batch run - parser.add_argument("-rb","--rbatch", help=f"run batch {Data.app_name}", action="store_true") + parser.add_argument("-rb","--rbatch", help=f"run batch {DataService.app_name}", action="store_true") # batch download parser.add_argument("-d","--download", help="Batch Download...", action="store_true") parser.add_argument("-net","--network", help="network checking...", action="store_true") diff --git a/src/machine.py b/src/machineService.py similarity index 82% rename from src/machine.py rename to src/machineService.py index 7e40db494628a8e420859baa1ce883f3dc805c18..87db0e9ebbedf523d651a1c177d1de8cc910da95 100644 --- a/src/machine.py +++ b/src/machineService.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from execute import Execute -from tool import Tool +from executeService import ExecuteService +from toolService import ToolService -class Machine: +class MachineService: def __init__(self): - self.exe = Execute() - self.tool = Tool() + self.exe = ExecuteService() + self.tool = ToolService() self.info2cmd = { 'CHECK network adapter':'nmcli d', 'CHECK Machine Bits':'getconf LONG_BIT', diff --git a/src/runService.py b/src/runService.py new file mode 100644 index 0000000000000000000000000000000000000000..13fcd38eaee4910f8c935c12d40b735c34cf9244 --- /dev/null +++ b/src/runService.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import sys +from toolService import ToolService +from executeService import ExecuteService +from dataService import DataService + +class RunService: + def __init__(self): + self.hpc_data = DataService() + self.exe = ExecuteService() + self.tool = ToolService() + self.ROOT = os.getcwd() + self.avail_ips_list = self.tool.gen_list(DataService.avail_ips) + + def gen_hostfile(self, nodes): + length = len(self.avail_ips_list) + if nodes > length: + print(f"You don't have {nodes} nodes, only {length} nodes available!") + sys.exit() + if nodes <= 1: + return + gen_nodes = '\n'.join(self.avail_ips_list[:nodes]) + print(f"HOSTFILE\n{gen_nodes}\nGENERATED.") + self.tool.write_file('hostfile', gen_nodes) + + # single run + def run(self): + print(f"start run {DataService.app_name}") + nodes = int(DataService.run_cmd['nodes']) + self.gen_hostfile(nodes) + run_cmd = self.hpc_data.get_run_cmd() + self.exe.exec_raw(run_cmd) + + def batch_run(self): + batch_file = 'batch_run.sh' + batch_file_path = os.path.join(self.ROOT, batch_file) + print(f"start batch run {DataService.app_name}") + batch_content = f''' +cd {DataService.case_dir} +{DataService.batch_cmd} +''' + self.tool.write_file(batch_file_path, batch_content) + run_cmd = f''' +chmod +x {batch_file} +./{batch_file} +''' + self.exe.exec_raw(run_cmd) diff --git a/src/testService.py b/src/testService.py new file mode 100644 index 0000000000000000000000000000000000000000..8ee2da0f9993540cc8956efd7dcc64616e55fa27 --- /dev/null +++ b/src/testService.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +from executeService import ExecuteService + +class TestService: + def __init__(self): + self.exe = ExecuteService() + self.ROOT = os.getcwd() + self.test_dir = os.path.join(self.ROOT, 'test') + + def test(self): + run_cmd = f''' +cd {self.test_dir} +./test-qe.sh +cd {self.test_dir} +./test-util.sh +''' + self.exe.exec_raw(run_cmd) + diff --git a/src/tool.py b/src/toolService.py similarity index 97% rename from src/tool.py rename to src/toolService.py index 7cd3b62251641058efc57e3732ec8ff07144bed0..1745b965bf1b745498f185d288b2d2173e9be276 100644 --- a/src/tool.py +++ b/src/toolService.py @@ -3,7 +3,7 @@ import time import os -class Tool: +class ToolService: def __init__(self): pass