diff --git a/build.sh b/build.sh index e66d9e7872a797bcea23f5f5a4ce9cac7192489a..da93d9b3745fea57b6339d68f02db1975095c61a 100644 --- a/build.sh +++ b/build.sh @@ -16,10 +16,13 @@ bash "${current_dir}"/tools/download_and_deploy/build_download.sh bash "${current_dir}"/tools/download_and_deploy/build_install.sh +bash "${current_dir}"/tools/download_and_deploy/build_pipeline_script_generator.sh + cd "${current_dir}"/build mkdir -p "${current_dir}"/build/devkit-pipeline-${tag}/linux cp -rf "${current_dir}"/build/deploy_tool/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux cp -rf "${current_dir}"/build/download_tool/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux +cp -rf "${current_dir}"/build/script_gen_dependency/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux tar -zcvf devkit-pipeline-${tag}.tar.gz devkit-pipeline-${tag} diff --git a/tools/download_and_deploy/src/handler/generate_pipeline_script.py b/tools/download_and_deploy/src/handler/generate_pipeline_script.py index 95fcc55e7599262616a04b8f07895abe27bc7979..335920bbb671d2ba9ff892494c56f1b832495b20 100644 --- a/tools/download_and_deploy/src/handler/generate_pipeline_script.py +++ b/tools/download_and_deploy/src/handler/generate_pipeline_script.py @@ -1,10 +1,13 @@ from handler.handler_and_node import Handler from pipeline_script_generator.jenkins_generator import JenkinsScript +from pipeline_script_generator.gitlab_generator import GitlabScript +generator_class = (JenkinsScript, GitlabScript) class GeneratePipelineScript(Handler): def handle(self, data) -> bool: - j = JenkinsScript(data) - with open("./xxx.jenkins", "w+", encoding="utf8") as f: - f.write(j.generate()) + for klass in generator_class: + o = klass(data) + with open("./script." + o.name, "w+", encoding="utf8") as f: + f.write(o.generate()) return True diff --git a/tools/download_and_deploy/src/pipeline_script_generator/gitlab_generator.py b/tools/download_and_deploy/src/pipeline_script_generator/gitlab_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..7d9ba6663accb7f266c7d837f759aeaf5131dd83 --- /dev/null +++ b/tools/download_and_deploy/src/pipeline_script_generator/gitlab_generator.py @@ -0,0 +1,232 @@ +from pipeline_script_generator.script_generator import ScriptGenerator + + +class GitlabScript(ScriptGenerator): + name = "gitlab" + base_template = """ +stages: + - migrating-applications + - affinity-analysis + - build + - test + - clamav + +variables: + # 源码迁移参数 + # 源码构建命令,在服务器中正常执行的构建命令,命令中如有空格,要使用单引号包住 + SOURCE_CODE_COMMAND: "make" + + # 软件迁移评估参数 + # 被扫描的软件包的路径 + SOFTWARE_PATH: "" + + # 字节对齐检查参数 + # 源码构建命令,在服务器中正常执行的构建命令,命令中如有空格,要使用单引号包住 + BYTE_ALIGNMENT_COMMAND: "make" + # 构建工具,当前工具支持make、cmake和automake,默认选项为make + BYTE_ALIGNMENT_TOOL: "make" + + # 内存一致性检查参数 + # BC文件路径 + MEMORY_BC_FILE: "" + + # 向量化检查参数 + # BC文件路径 + VECTORIZED_BC_FILE: "" + # 源码构建命令,在服务器中正常执行的构建命令,命令中如有空格,要使用单引号包住 + VECTORIZED_COMMAND: "make" + + # 编译参数 + # 编译命令 + BUILD_COMMAND: "" + + # Java Performance Analysis功能参数 + # 需要采集的目标程序所在的服务器地址, 多个使用逗号隔离 + JAVA_TARGET_SERVER_IP: "" + # 需要采集的应用名称,多个采用逗号隔离 + JAVA_APPLICATION_NAME: "" + # 采集目标应用时间,单位秒。当存在-j参数时,jmeter结束或者到达采集执行时间,结束采集 + JAVA_COLLECTION_APPLICATION_DURATION: "" + # jmeter执行命令。例如:bash /opt/apache-jmeter-5.6.3/bin/jmeter.sh -nt /home/xxx/Request.jmx -l /home/xxx/result.html -eo /home/xxx/report + JAVA_JMETER_COMMAND: "" + # jmeter -l参数,将测试结果以指定的文件名保存到本地 + JAVA_JMETER_RESULT_FILE_NAME: "" + # jmeter -o参数,jmeter运行结束后生成的测试报告文件路径 + JAVA_JMETER_RESULT_REPORT_DIR: "" + # Devkit工具部署的环境IP + DEVKIT_WEB_IP: "" + +##STAGES## +""" + source_migration_template = """ +# 源码迁移 +source-code-migration: + stage: migrating-applications + tags: + - kunpeng_scanner # 对应gitlab-runner注册时的标签,可选择多个 + script: + - echo '====== 源码迁移 ======' + - devkit porting src-mig -i ./ -c $SOURCE_CODE_COMMAND -r html || [ $? -eq 1 ] && echo 'Warning:扫描报告包含建议项' + - mv ./src-mig*.html ./SourceCodeScanningReport.html + artifacts: + paths: + - SourceCodeScanningReport.html + name: src-mig +""" + package_migration_template = """ +# 软件迁移评估 +software-migration-assessment: + stage: migrating-applications + tags: + - kunpeng_scanner # 对应gitlab-runner注册时的标签,可选择多个 + script: + - echo '====== 软件迁移评估 ======' + - ls /usr/local/bin/devkit + - devkit porting pkg-mig -i $SOFTWARE_PATH -r html || [ $? -eq 1 ] && echo 'Warning:扫描报告包含建议项' + - mv ./pkg-mig*.html ./SoftwareMigrationAssessment.html + artifacts: + paths: + - SoftwareMigrationAssessment.html + name: pkg-mig +""" + mode_check_template = """ +# 64位运行模式检查 +64-bit-running-mode-check: + stage: affinity-analysis + tags: + - kunpeng_scanner # 对应gitlab-runner注册时的标签,可选择多个 + script: + - echo '====== 64位运行模式检查 ======' + - devkit advisor mode-check -i ./ -r html + - mv ./mode_check*.html ./64-bit-running-mode-check.html + artifacts: + paths: + - 64-bit-running-mode-check.html + name: mode-check +""" + byte_alignment_template = """ +# 字节对齐检查 +byte-alignment-check: + stage: affinity-analysis + tags: + - kunpeng_scanner # 对应gitlab-runner注册时的标签,可选择多个 + script: + - echo '====== 字节对齐检查 ======' + - devkit advisor byte-align -i ./ -c $BYTE_ALIGNMENT_COMMAND -b $BYTE_ALIGNMENT_TOOL -r html + - mv ./byte-align*.html ./byte-alignment-check.html + artifacts: + paths: + - byte-alignment-check.html + name: byte-align +""" + memory_consistency_template = """ +# 内存一致性检查 +memory-consistency-check: + stage: affinity-analysis + tags: + - kunpeng_scanner # 对应gitlab-runner注册时的标签,可选择多个 + script: + - echo '====== 内存一致性检查 ======' + # 需编写生成的BC文件脚本 + - devkit advisor mem-cons -i ./ -f $MEMORY_BC_FILE -r html + - mv ./mem-cons*.html ./memory-consistency-check.html + artifacts: + paths: + - memory-consistency-check.html + name: mem-cons +""" + vector_check_template = """ +# 向量化检查 +vectorized-check: + stage: affinity-analysis + tags: + - kunpeng_scanner # 对应gitlab-runner注册时的标签,可选择多个 + script: + - echo '====== 向量化检查 ======' + # 需编写生成的BC文件脚本; + - devkit advisor vec-check -i ./ -f $VECTORIZED_BC_FILE -c $VECTORIZED_COMMAND -r html + - mv ./vec-check*.html ./vectorized-check.html + artifacts: + paths: + - vectorized-check.html + name: vec-check +""" + gcc_template = """ +# 普通编译 +build: + stage: build + tags: + - kunpeng_c_builder_gcc # 对应gitlab-runner注册时的标签,可选择多个 + script: + - $BUILD_COMMAND +""" + bisheng_compiler_template = """ +# 普通编译 +build: + stage: build + tags: + - kunpeng_c_builder_bisheng_compiler # 对应gitlab-runner注册时的标签,可选择多个 + script: + - $BUILD_COMMAND +""" + java8_build_template = """ +# 普通编译 +build: + stage: build + tags: + - kunpeng_java_builder_jdk8 # 对应gitlab-runner注册时的标签,可选择多个 + script: + - $BUILD_COMMAND +""" + java17_build_template = """ +# 普通编译 +build: + stage: build + tags: + - kunpeng_java_builder_jdk17 # 对应gitlab-runner注册时的标签,可选择多个 + script: + - $BUILD_COMMAND +""" + a_fot_template = """ +# 使用GCC for openEuler编译,使用A-FOT工具时,需要根据用户指南填写配置项 +A-FOT: + stage: build + tags: + - kunpeng_c_builder_gcc # 对应gitlab-runner注册时的标签,可选择多个 + script: + - export PATH=${HOME}/.local/gcc-10.3.1-2023.12-aarch64-linux/bin:$PATH + - a-fot --config_file a-fot.ini +""" + java_perf_template = "" + compatibility_test_template = """ +# 鲲鹏兼容测试 +compatibility_test: # This job runs in the build stage, which runs first. + stage: test + tags: + - kunpeng_executor # 对应gitlab-runner注册时的标签,可选择多个 + script: + - CURDIR=$(pwd) + - echo $CURDIR + - /bin/cp -rf /root/.local/compatibility_testing/template.html.bak /root/.local/compatibility_testing/template.html + - /bin/bash compatibility_test + - /bin/cp -rf /root/.local/compatibility_testing/compatibility_report.html $CURDIR/compatibility_report.html + - sudo /bin/bash /root/.local/compatibility_testing/report_result.sh + - echo "请去 '${CURDIR}'/compatibility_report.html 查看报告 " + artifacts: + paths: + - compatibility_report.html + name: compatibility_report +""" + clamav_template = """ +# 病毒扫描 +clamscan: + stage: build + tags: + - kunpeng_clamav # 对应gitlab-runner注册时的标签,可选择多个 + script: + - clamscan -i -r ./ -l ./clamscan.log + artifacts: + paths: + - clamscan.log + name: clamscan +""" \ No newline at end of file diff --git a/tools/download_and_deploy/src/pipeline_script_generator/jenkins_generator.py b/tools/download_and_deploy/src/pipeline_script_generator/jenkins_generator.py index 89c2bccefcaebb1d79b333cc11a28657949fc6b8..50045224ffc703d59c1289e7dc32d3866be69c37 100644 --- a/tools/download_and_deploy/src/pipeline_script_generator/jenkins_generator.py +++ b/tools/download_and_deploy/src/pipeline_script_generator/jenkins_generator.py @@ -2,7 +2,19 @@ from pipeline_script_generator.script_generator import ScriptGenerator class JenkinsScript(ScriptGenerator): + name = "jenkins" base_template = """ +def get_code(GIT_BRANCH, GIT_TARGET_DIR_NAME, GIT_URL) { + sh ''' + rm -fr "${GIT_TARGET_DIR_NAME}" + ''' + checkout scmGit(branches: [[name: "*/${GIT_BRANCH}"]], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: "${GIT_TARGET_DIR_NAME}"], cleanBeforeCheckout(deleteUntrackedNestedRepositories: true)],userRemoteConfigs: [[url: "${GIT_URL}"]]) + sh ''' + rm -rf ./report_dir + mkdir ./report_dir + ''' +} + pipeline { agent none options { @@ -61,26 +73,17 @@ pipeline { // 编译参数 // 编译命令 - BUILD_COMMAND = "" + BUILD_COMMAND = ''' + + ''' + //A-FOT配置文件存放的路径 + A_FOT_CONF_PATH = "" + + //病毒扫描路径 + CLAMAV_PATH = "" } stages{ - // 获取源码 - stage('git-clone-code') { - steps { - checkout scmGit(branches: [[name: "*/${GIT_BRANCH}"]], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: "${GIT_TARGET_DIR_NAME}"], cleanBeforeCheckout(deleteUntrackedNestedRepositories: true)], userRemoteConfigs: [[credentialsId: "${CREDENTIALS_ID}", url: "${GIT_URL}"]]) - } - } - // 创建门禁报告存储路径 - stage('mkdir-devkit-report') { - steps { - sh ''' - rm -rf ./report_dir - mkdir ./report_dir - ''' - } - - } ##STAGES## } } @@ -92,6 +95,7 @@ pipeline { label 'kunpeng_scanner' } steps { + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") sh ''' /usr/bin/rm -rf ./report_dir/*.html ''' @@ -218,6 +222,7 @@ pipeline { label 'kunpeng_scanner' } steps { + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") sh ''' /usr/bin/rm -rf ./report_dir/*.html ''' @@ -272,7 +277,8 @@ pipeline { label 'kunpeng_scanner' } steps { - sh ''' + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") + sh ''' /usr/bin/rm -rf ./report_dir/*.html ''' script{ @@ -326,6 +332,7 @@ pipeline { label 'kunpeng_scanner' } steps { + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") sh ''' /usr/bin/rm -rf ./report_dir/*.html ''' @@ -380,6 +387,7 @@ pipeline { label 'kunpeng_scanner' } steps { + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") sh ''' /usr/bin/rm -rf ./report_dir/*.html ''' @@ -427,16 +435,39 @@ pipeline { } } """ - build_template = """ + java8_build_template = """ + // 普通编译 + stage('java8-build') { + agent { + label 'kunpeng_java_builder_jdk8' + } + steps { + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") + sh "${BUILD_COMMAND}" + } + } +""" + java17_build_template = """ + // 普通编译 + stage('java17-build') { + agent { + label 'kunpeng_java_builder_jdk17' + } + steps { + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") + sh "${BUILD_COMMAND}" + } + } +""" + gcc_template = """ // 普通编译 stage('gcc-build') { agent { label 'kunpeng_c_builder_gcc' } steps { - sh ''' - "${BUILD_COMMAND}" - ''' + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") + sh "${BUILD_COMMAND}" } } """ @@ -447,9 +478,10 @@ pipeline { label 'kunpeng_c_builder_bisheng_compiler' } steps { - sh ''' source ${HOME}/.local/wrap-bin/devkit_pipeline.sh - "${BUILD_COMMAND}" - ''' + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") + sh ''' source ${HOME}/.local/wrap-bin/devkit_pipeline.sh ''' + sh "${BUILD_COMMAND}" + } } """ @@ -460,43 +492,16 @@ pipeline { label 'kunpeng_c_builder_gcc' } steps { + get_code("${GIT_BRANCH}", "${GIT_TARGET_DIR_NAME}", "${GIT_URL}") sh ''' export PATH=${HOME}/.local/gcc-10.3.1-2023.12-aarch64-linux/bin:$PATH - a-fot --config_file a-fot.ini + a-fot --config_file "${A_FOT_CONF_PATH}"/a-fot.ini ''' } } """ java_perf_template = """ - // 鲲鹏兼容测试 - stage('compatibility_test') { - agent { - label 'kunpeng_executor' - } - steps { - script{ - echo '====== lkp test ======' - sh ''' - CURDIR=$(pwd) - cp -rf ${HOME}/.local/compatibility_testing/template.html.bak ${HOME}/.local/compatibility_testing/template.html - sh compatibility_test - cp -rf ${HOME}/.local/compatibility_testing/compatibility_report.html $CURDIR - ''' - sh(script: "sudo bash ${HOME}/.local/compatibility_testing/report_result.sh", returnStdout:true).trim() // 这个是用于判断lkp 命令后生成的结果是否符合预期,需要根据不同的run脚本生成的结果文件去做不同的结果判断结果 - } - } - post { - always { - publishHTML(target: [allowMissing: false, - alwaysLinkToLastBuild: false, - keepAll : true, - reportDir : '.', - reportFiles : 'compatibility_report.html', - reportName : 'compatibility test Report'] - ) - } - } - } + """ compatibility_test_template = """ // 鲲鹏兼容测试 @@ -506,14 +511,13 @@ pipeline { } steps { script{ - echo '====== lkp test ======' sh ''' CURDIR=$(pwd) cp -rf ${HOME}/.local/compatibility_testing/template.html.bak ${HOME}/.local/compatibility_testing/template.html - sh compatibility_test + ${HOME}/.local/compatibility_testing/bin/compatibility_test cp -rf ${HOME}/.local/compatibility_testing/compatibility_report.html $CURDIR ''' - sh(script: "sudo bash ${HOME}/.local/compatibility_testing/report_result.sh", returnStdout:true).trim() // 这个是用于判断lkp 命令后生成的结果是否符合预期,需要根据不同的run脚本生成的结果文件去做不同的结果判断结果 + sh(script: "sudo bash ${HOME}/.local/compatibility_testing/report_result.sh", returnStdout:true).trim() } } post { @@ -529,4 +533,17 @@ pipeline { } } """ - clamav_template = "" \ No newline at end of file + + clamav_template = """ + //病毒扫描 + stage('clamav') { + agent {label 'kunpeng_clamav'} + steps { + sh ''' + freshclam + clamscan -i -r "${CLAMAV_PATH}" -l clamav.log + ''' + } + + } +""" \ No newline at end of file diff --git a/tools/download_and_deploy/src/pipeline_script_generator/script_generator.py b/tools/download_and_deploy/src/pipeline_script_generator/script_generator.py index d54f761614c8266b714718d5e52290594eb4b106..18bdec5a9ba81c033fcb65d3a7f2d10375e92bd7 100644 --- a/tools/download_and_deploy/src/pipeline_script_generator/script_generator.py +++ b/tools/download_and_deploy/src/pipeline_script_generator/script_generator.py @@ -2,6 +2,7 @@ from constant import * class ScriptGenerator: + name = "" base_template = "" source_migration_template = "" package_migration_template = "" @@ -9,8 +10,10 @@ class ScriptGenerator: byte_alignment_template = "" memory_consistency_template = "" vector_check_template = "" - build_template = "" + gcc_template = "" bisheng_compiler_template = "" + java8_build_template = "" + java17_build_template = "" a_fot_template = "" java_perf_template = "" compatibility_test_template = "" @@ -27,9 +30,11 @@ class ScriptGenerator: self.memory_consistency_template, self.vector_check_template ], - C_BUILDER_GCC: [self.build_template, self.a_fot_template], + C_BUILDER_GCC: [self.gcc_template, self.a_fot_template], C_BUILDER_BISHENG_COMPILER: [self.bisheng_compiler_template], - EXECUTOR: [self.compatibility_test_template, self.java_perf_template], + JAVA_BUILDER_JDK8: [self.java8_build_template], + JAVA_BUILDER_JDK17: [self.java17_build_template], + COMPATIBILITY: [self.compatibility_test_template], CLAMAV: [self.clamav_template] }