diff --git a/.gitignore b/.gitignore index 5fd13d867676e4f88848768fac5d25e9e8ab4212..7a28623d77dde33709f918e67b23b8932ffc80ec 100644 --- a/.gitignore +++ b/.gitignore @@ -3,13 +3,10 @@ build .idea .venv __pycache__ -tools/install_dependency/src/component -component/LkpTests/compatibility_help/compatibility_testing -component/LkpTests/compatibility_help/compatibility_testing.tar.gz -component/LkpTests/lkp_help/lkp-tests.tar.gz +devkitdependencies + component/DevKitTester/devkit_tester/config/log.ini component/DevKitTester/devkit_tester/config/devkit_distribute.yaml -devkitdependencies component/DevKitTester/JFRParser/.idea component/DevKitTester/JFRParser/.gitignore component/DevKitTester/JFRParser/target \ No newline at end of file diff --git a/build.sh b/build.sh index 48cc47d41f6c153a830b5f94d8f3002073c02667..e66d9e7872a797bcea23f5f5a4ce9cac7192489a 100644 --- a/build.sh +++ b/build.sh @@ -5,20 +5,21 @@ set -e current_dir=$(cd $(dirname "$0"); pwd) -tag="v1.0" +tag="v1.0.1" +rm -rf "${current_dir}"/build mkdir -p "${current_dir}"/build bash "${current_dir}"/component/build_component.sh -bash "${current_dir}"/tools/install_dependency/build_download.sh +bash "${current_dir}"/tools/download_and_deploy/build_download.sh -bash "${current_dir}"/tools/install_dependency/build_install.sh +bash "${current_dir}"/tools/download_and_deploy/build_install.sh cd "${current_dir}"/build mkdir -p "${current_dir}"/build/devkit-pipeline-${tag}/linux -cp -rf "${current_dir}"/build/install_dependency/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux -cp -rf "${current_dir}"/build/download_dependency/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux +cp -rf "${current_dir}"/build/deploy_tool/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux +cp -rf "${current_dir}"/build/download_tool/dist/* "${current_dir}"/build/devkit-pipeline-${tag}/linux tar -zcvf devkit-pipeline-${tag}.tar.gz devkit-pipeline-${tag} diff --git a/component/BiShengCompiler/install.sh b/component/BiShengCompiler/install.sh index 999e89296f3418571da1a0ef6226e58df8e5c5c0..c05d5b33ce858db79c9adc25e6e7d0f5af5b0cb0 100644 --- a/component/BiShengCompiler/install.sh +++ b/component/BiShengCompiler/install.sh @@ -1,34 +1,29 @@ #!/bin/bash cd /tmp/devkitdependencies/ -verify_signature=$(sha256sum -c BiShengCompiler-3.2.0-aarch64-linux.tar.gz.sha256 >/dev/null 2>&1; echo $?) -if [[ ${verify_signature} -eq "0" ]]; then - if [[ ! -d ${HOME}/.local/BiShengCompiler-3.2.0-aarch64-linux ]]; then - mkdir -p ${HOME}/.local - echo "Decompress BiShengCompiler-3.2.0-aarch64-linux.tar.gz to ${HOME}/.local" - tar --no-same-owner -zxf /tmp/devkitdependencies/BiShengCompiler-3.2.0-aarch64-linux.tar.gz -C ${HOME}/.local - echo "Decompress BiShengCompiler-3.2.0-aarch64-linux.tar.gz to ${HOME}/.local finished." - fi +if [[ ! -d ${HOME}/.local/BiShengCompiler-3.2.0-aarch64-linux ]]; then + mkdir -p ${HOME}/.local - clang_path=$(which clang) - if [[ ${clang_path} != ${HOME}/.local/BiShengCompiler-3.2.0-aarch64-linux/bin/clang ]]; then - sed -i '/.*export BISHENG_COMPILER_HOME=${HOME}\/.local/d' ${HOME}/.bashrc - sed -i '/.*export .*=${BISHENG_COMPILER_HOME}.*/d' ${HOME}/.bashrc + echo "Decompress BiShengCompiler-3.2.0-aarch64-linux.tar.gz to ${HOME}/.local" + tar --no-same-owner -zxf /tmp/devkitdependencies/BiShengCompiler-3.2.0-aarch64-linux.tar.gz -C ${HOME}/.local + echo "Decompress BiShengCompiler-3.2.0-aarch64-linux.tar.gz to ${HOME}/.local finished." +fi - echo "change ${HOME}/.bashrc" - cat >> ${HOME}/.bashrc <<'EOF' +clang_path=$(which clang) +if [[ ${clang_path} != ${HOME}/.local/BiShengCompiler-3.2.0-aarch64-linux/bin/clang ]]; then + sed -i '/.*export BISHENG_COMPILER_HOME=${HOME}\/.local/d' ${HOME}/.bashrc + sed -i '/.*export .*=${BISHENG_COMPILER_HOME}.*/d' ${HOME}/.bashrc + + echo "change ${HOME}/.bashrc" + cat >> ${HOME}/.bashrc <<'EOF' export BISHENG_COMPILER_HOME=${HOME}/.local/BiShengCompiler-3.2.0-aarch64-linux export PATH=${BISHENG_COMPILER_HOME}/bin:${PATH} export LD_LIBRARY_PATH=${BISHENG_COMPILER_HOME}/lib:${BISHENG_COMPILER_HOME}/lib/aarch64-unknown-linux-gnu:${LD_LIBRARY_PATH} EOF - echo "source ${HOME}/.bashrc" - set +x - source ${HOME}/.bashrc - else - echo "install BiShengCompiler-3.2.0 success." - fi - + echo "source ${HOME}/.bashrc" + set +x + source ${HOME}/.bashrc else - echo "Failed to verify the signature of the BiShengCompiler-3.2.0-aarch64-linux.tar.gz installation package." -fi \ No newline at end of file + echo "install BiShengCompiler-3.2.0 success." +fi diff --git a/component/BiShengJDK17/install.sh b/component/BiShengJDK17/install.sh index e599921a2a648d1d3eb01deb0316279b91cb5159..732fd0bf2b1445b0ed0846da44759f88f20dc9c3 100644 --- a/component/BiShengJDK17/install.sh +++ b/component/BiShengJDK17/install.sh @@ -1,15 +1,11 @@ #!/bin/bash cd /tmp/devkitdependencies/ -verify_signature=$(sha256sum -c bisheng-jdk-17.0.10-linux-aarch64.tar.gz.sha256 >/dev/null 2>&1; echo $?) -if [[ ${verify_signature} -eq "0" ]]; then - if [[ ! -d ${HOME}/.local/bisheng-jdk-17.0.10 ]]; then - mkdir -p ${HOME}/.local - echo "Decompress bisheng-jdk-17.0.10-linux-aarch64.tar.gz to ${HOME}/.local" - tar --no-same-owner -zxf /tmp/devkitdependencies/bisheng-jdk-17.0.10-linux-aarch64.tar.gz -C ${HOME}/.local - echo "Decompress bisheng-jdk-17.0.10-linux-aarch64.tar.gz to ${HOME}/.local finished." - fi -else - echo "Failed to verify the signature of the bisheng-jdk-17.0.10-linux-aarch64.tar.gz installation package." +if [[ ! -d ${HOME}/.local/bisheng-jdk-17.0.10 ]]; then + mkdir -p ${HOME}/.local + + echo "Decompress bisheng-jdk-17.0.10-linux-aarch64.tar.gz to ${HOME}/.local" + tar --no-same-owner -zxf /tmp/devkitdependencies/bisheng-jdk-17.0.10-linux-aarch64.tar.gz -C ${HOME}/.local + echo "Decompress bisheng-jdk-17.0.10-linux-aarch64.tar.gz to ${HOME}/.local finished." fi diff --git a/component/BiShengJDK8/install.sh b/component/BiShengJDK8/install.sh index 2e8990c6ff16a4cea26d6e10587ea13aa6ea33a6..2e015e75914d065c22ed30227e6d900d4d8906d4 100644 --- a/component/BiShengJDK8/install.sh +++ b/component/BiShengJDK8/install.sh @@ -1,15 +1,11 @@ #!/bin/bash cd /tmp/devkitdependencies/ -verify_signature=$(sha256sum -c bisheng-jdk-8u402-linux-aarch64.tar.gz.sha256 >/dev/null 2>&1; echo $?) -if [[ ${verify_signature} -eq "0" ]]; then - if [[ ! -d ${HOME}/.local/bisheng-jdk1.8.0_402 ]]; then - mkdir -p ${HOME}/.local - echo "Decompress bisheng-jdk-8u402-linux-aarch64.tar.gz to ${HOME}/.local" - tar --no-same-owner -zxf /tmp/devkitdependencies/bisheng-jdk-8u402-linux-aarch64.tar.gz -C ${HOME}/.local - echo "Decompress bisheng-jdk-8u402-linux-aarch64.tar.gz to ${HOME}/.local finished." - fi -else - echo "Failed to verify the signature of the bisheng-jdk-8u402-linux-aarch64.tar.gz installation package." +if [[ ! -d ${HOME}/.local/bisheng-jdk1.8.0_402 ]]; then + mkdir -p ${HOME}/.local + + echo "Decompress bisheng-jdk-8u402-linux-aarch64.tar.gz to ${HOME}/.local" + tar --no-same-owner -zxf /tmp/devkitdependencies/bisheng-jdk-8u402-linux-aarch64.tar.gz -C ${HOME}/.local + echo "Decompress bisheng-jdk-8u402-linux-aarch64.tar.gz to ${HOME}/.local finished." fi diff --git a/component/clamAV/check_install_result.sh b/component/ClamAV/check_install_result.sh similarity index 100% rename from component/clamAV/check_install_result.sh rename to component/ClamAV/check_install_result.sh diff --git a/component/clamAV/install.sh b/component/ClamAV/install.sh similarity index 100% rename from component/clamAV/install.sh rename to component/ClamAV/install.sh diff --git a/component/CloudTest/pom.xml b/component/CloudTest/pom.xml deleted file mode 100644 index f9929bf23a9de54a92e104a59e2a6c667fc726ed..0000000000000000000000000000000000000000 --- a/component/CloudTest/pom.xml +++ /dev/null @@ -1,260 +0,0 @@ - - - 4.0.0 - - org.springframework.boot - spring-boot-starter-parent - 2.7.15 - - - com.huawei - cloudTest - 1.0.0 - cloudTest - Demo project for Spring Boot - - 1.8 - 2021.0.3 - 4.1.100.Final - - - - com.mikesamuel - json-sanitizer - 1.2.3 - - - - org.springframework.boot - spring-boot-starter-web - - - - org.apache.commons - commons-lang3 - 3.12.0 - - - org.apache.commons - commons-compress - 1.23.0 - - - commons-io - commons-io - 2.11.0 - - - commons-fileupload - commons-fileupload - 1.5 - - - org.apache.httpcomponents - httpclient - 4.5.13 - - - - org.mybatis.spring.boot - mybatis-spring-boot-starter - 2.2.2 - - - - org.mariadb.jdbc - mariadb-java-client - - - org.springframework.boot - spring-boot-starter-test - test - - - junit - junit - test - - - com.alibaba - fastjson - 1.2.83 - - - org.projectlombok - lombok - - - org.springframework.boot - spring-boot-starter-data-redis - - - io.netty - netty-buffer - - - io.netty - netty-codec - - - io.netty - netty-common - - - io.netty - netty-handler - - - io.netty - netty-resolver - - - io.netty - netty-transport - - - io.netty - netty-transport-native-unix-common - - - - - io.netty - netty-buffer - ${netty.version} - - - io.netty - netty-codec - ${netty.version} - - - io.netty - netty-common - ${netty.version} - - - io.netty - netty-handler - ${netty.version} - - - io.netty - netty-resolver - ${netty.version} - - - io.netty - netty-transport - ${netty.version} - - - io.netty - netty-transport-native-unix-common - ${netty.version} - - - - - org.apache.sshd - sshd-core - 2.10.0 - - - com.jcraft - jsch - 0.1.54 - - - org.springframework.cloud - spring-cloud-starter-openfeign - 4.0.3 - - - org.bouncycastle - bcpkix-jdk15on - - - org.springframework.security - spring-security-crypto - - - - - com.github.tobato - fastdfs-client - 1.27.2 - - - commons-beanutils - commons-beanutils - - - - - commons-beanutils - commons-beanutils - 1.9.4 - - - - - - org.yaml - snakeyaml - 1.32 - - - - xyz.capybara - clamav-client - 2.1.2 - - - - - - - - - org.springframework.cloud - spring-cloud-dependencies - ${spring-cloud.version} - pom - import - - - - - - cloudTest - - - org.apache.maven.plugins - maven-dependency-plugin - 3.1.2 - - - copy-dependencies - compile - - copy-dependencies - - - ${project.build.directory}/${project.build.finalName}/WEB-INF/lib - - system - - - - - - org.springframework.boot - spring-boot-maven-plugin - 2.7.15 - - - - - diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/CompatibleTestApplication.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/CompatibleTestApplication.java deleted file mode 100644 index 331e92c0f8ae3c63915071d1505884209575c08e..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/CompatibleTestApplication.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; -import org.springframework.cloud.openfeign.EnableFeignClients; -import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.scheduling.annotation.EnableScheduling; - -/** - * CompatibleTestApplication - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class}) -@EnableAsync -@EnableScheduling -@EnableFeignClients -public class CompatibleTestApplication { - /** - * main - * - * @param args params - */ - public static void main(String[] args) { - SpringApplication.run(CompatibleTestApplication.class, args); - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/config/TaskPoolConfig.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/config/TaskPoolConfig.java deleted file mode 100644 index 45eb9662362f86d84367c036eef22359f0b15e1c..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/config/TaskPoolConfig.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.common.config; - -import lombok.extern.slf4j.Slf4j; - -import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.AsyncConfigurer; -import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.concurrent.Executor; - -/** - * TaskPoolConfig - * - * @author kongcaizhi - * @since 2022-10-19 - */ -@Configuration -@EnableAsync -@Slf4j -public class TaskPoolConfig implements AsyncConfigurer { - @Override - @Bean(name = "asyncExecutor", destroyMethod = "shutdown") - public Executor getAsyncExecutor() { - ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - - // 核心线程数 - executor.setCorePoolSize(100); - - // 最大线程数,默认 40000 - executor.setMaxPoolSize(300); - - // 线程池队列最大线程数,默认80000 - executor.setQueueCapacity(1000); - - // 线程池线程最大空闲时间（秒） - executor.setKeepAliveSeconds(300); - executor.setThreadNamePrefix("async-task-"); - executor.initialize(); - return executor; - } - - @Override - public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() { - return (throwable, method, objects) -> - log.error( - "An exception occurred during asynchronous task execution, message {}, method {}, params {}", - throwable.getLocalizedMessage(), - method, - objects - ); - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/exception/BaseException.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/exception/BaseException.java deleted file mode 100644 index 755b5ffac02f4fb78fc83918db47283ec0c19719..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/exception/BaseException.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.common.exception; - -/** - * BaseException - * - * @author kongcaizhi - * @since 2021-10-19 - */ -public class BaseException extends RuntimeException { - private static final long serialVersionUID = 120492404095565805L; - - /** - * BaseException - * - * @param message message - */ - public BaseException(String message) { - super(message); - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/exception/ClamAvSizeLimitException.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/exception/ClamAvSizeLimitException.java deleted file mode 100644 index c0f310f636674d07d08d48db60109c040a627c8d..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/exception/ClamAvSizeLimitException.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.common.exception; - -/** - * ClamAVSizeLimitException - * - * @author kongcaizhi - * @since 2021-10-19 - */ -public class ClamAvSizeLimitException extends RuntimeException { - private static final long serialVersionUID = 1L; - - /** - * Construction function - * - * @param msg message - */ - public ClamAvSizeLimitException(String msg) { - super(msg); - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/exception/SshErrorException.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/exception/SshErrorException.java deleted file mode 100644 index b011a75f9d53ebe8889687f926fc7eb0d9178c88..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/exception/SshErrorException.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.common.exception; - -/** - * ClamAVSizeLimitException - * - * @author kongcaizhi - * @since 2021-10-19 - */ -public class SshErrorException extends Exception { - private static final long serialVersionUID = 116717817166294456L; - - /** - * SshErrorException - * - * @param message message - */ - public SshErrorException(String message) { - super(message); - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/typehandler/AbstractListTypeHandler.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/typehandler/AbstractListTypeHandler.java deleted file mode 100644 index 4ad6d098344d4b67f1c694bb595bda454b88614e..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/typehandler/AbstractListTypeHandler.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.common.typehandler; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.type.CollectionType; -import com.google.json.JsonSanitizer; - -import lombok.SneakyThrows; - -import org.apache.ibatis.type.BaseTypeHandler; -import org.apache.ibatis.type.JdbcType; -import org.apache.ibatis.type.MappedJdbcTypes; -import org.apache.ibatis.type.MappedTypes; -import org.springframework.util.StringUtils; - -import java.sql.CallableStatement; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -/** - * AbstractListTypeHandler - * - * @author kongcaizhi - * @since 2021-11-19 - */ -@MappedTypes(List.class) -@MappedJdbcTypes(JdbcType.VARCHAR) -public abstract class AbstractListTypeHandler extends BaseTypeHandler { - /** - * objectMapper - */ - protected ObjectMapper objectMapper = new ObjectMapper(); - - /** - * javaType - */ - protected CollectionType javaType; - - @SneakyThrows - @Override - public void setNonNullParameter(PreparedStatement ps, int i, List list, JdbcType jdbcType) throws SQLException { - ps.setString(i, objectMapper.writeValueAsString(list)); - } - - @SneakyThrows - @Override - public List getNullableResult(ResultSet rs, String columnName) throws SQLException { - String json = rs.getString(columnName); - List list = null; - if (!StringUtils.hasLength(json)) { - list = objectMapper.readValue(JsonSanitizer.sanitize(json), javaType); - } - return list; - } - - @SneakyThrows - @Override - public List getNullableResult(ResultSet rs, int columnIndex) throws SQLException { - String json = rs.getString(rs.getString(columnIndex)); - List list = null; - if (!StringUtils.hasLength(json)) { - list = objectMapper.readValue(JsonSanitizer.sanitize(json), javaType); - } - return list; - } - - @SneakyThrows - @Override - public List getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { - String json = cs.getString(columnIndex); - List list = null; - if (!StringUtils.hasLength(json)) { - list = objectMapper.readValue(JsonSanitizer.sanitize(json), javaType); - } - return list; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/typehandler/CryptoTypeHandler.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/typehandler/CryptoTypeHandler.java deleted file mode 100644 index fb2b2b6ca89cc4b87c919ad49522c56df4802103..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/typehandler/CryptoTypeHandler.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.common.typehandler; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.util.AESUtil; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; -import org.apache.ibatis.type.BaseTypeHandler; -import org.apache.ibatis.type.JdbcType; -import org.apache.ibatis.type.MappedJdbcTypes; - -import java.sql.CallableStatement; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; - -/** - * CryptoTypeHandler - * - * @author kongcaizhi - * @since 2021-11-19 - */ -@Slf4j -@MappedJdbcTypes(JdbcType.VARCHAR) -public class CryptoTypeHandler extends BaseTypeHandler { - private static final Integer PASSWORD_LENGTH_WITHOUT_ENCRYPT = 50; - - @Override - public void setNonNullParameter(PreparedStatement preparedStatement, int i, String s, JdbcType jdbcType) - throws SQLException { - preparedStatement.setString(i, doEncrypt(s)); - } - - @Override - public String getNullableResult(ResultSet resultSet, String s) throws SQLException { - return doDecruypt(resultSet.getString(s)); - } - - @Override - public String getNullableResult(ResultSet resultSet, int i) throws SQLException { - return doDecruypt(resultSet.getString(i)); - } - - @Override - public String getNullableResult(CallableStatement callableStatement, int i) throws SQLException { - return doDecruypt(callableStatement.getString(i)); - } - - private String doEncrypt(String password) { - return AESUtil.gcmEncrypt(password, AESUtil.CRYPTO.getAesDefaultKey(), AESUtil.CRYPTO.getAesDefaultAad()); - } - - private String doDecruypt(String password) { - if (!isEncrypt(password)) { - throw new BaseException("password decruypt failed"); - } - return AESUtil.gcmDecrypt(password, AESUtil.CRYPTO.getAesDefaultKey()); - } - - private boolean isEncrypt(String password) { - return (StringUtils.isNotEmpty(password) && password.length() > PASSWORD_LENGTH_WITHOUT_ENCRYPT) - ? true : false; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/typehandler/JacksonTypeHandler.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/typehandler/JacksonTypeHandler.java deleted file mode 100644 index 5fe1e0ef9b6413b68bc0181bff2bcc5713875839..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/common/typehandler/JacksonTypeHandler.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.common.typehandler; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.ibatis.exceptions.PersistenceException; -import org.apache.ibatis.type.BaseTypeHandler; -import org.apache.ibatis.type.JdbcType; -import org.apache.ibatis.type.MappedJdbcTypes; - -import java.io.IOException; -import java.sql.CallableStatement; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; - -/** - * JacksonTypeHandler - * - * @author kongcaizhi - * @since 2021-11-19 - */ -@Slf4j -@MappedJdbcTypes(JdbcType.VARCHAR) -public class JacksonTypeHandler extends BaseTypeHandler { - private static final ObjectMapper OBJECT_MAPPER; - - static { - OBJECT_MAPPER = new ObjectMapper(); - OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - } - - private final Class type; - - /** - * JacksonTypeHandler - * - * @param type Class - */ - public JacksonTypeHandler(Class type) { - if (log.isTraceEnabled()) { - log.trace("JacksonTypeHandler ({})", type); - } - if (type == null) { - throw new PersistenceException("Type argument cannot be null"); - } - this.type = type; - } - - private T parse(String json) { - T object = null; - try { - if (!(json == null || json.length() == 0)) { - object = OBJECT_MAPPER.readValue(json, type); - } - return object; - } catch (IOException e) { - log.error(e.getMessage(), e); - throw new BaseException(e.getMessage()); - } - } - - private String toJsonString(T obj) { - try { - return OBJECT_MAPPER.writeValueAsString(obj); - } catch (JsonProcessingException e) { - log.error(e.getMessage(), e); - throw new BaseException(e.getMessage()); - } - } - - @Override - public void setNonNullParameter(PreparedStatement ps, int i, T parameter, JdbcType jdbcType) throws SQLException { - ps.setString(i, toJsonString(parameter)); - } - - @Override - public T getNullableResult(ResultSet rs, String columnName) throws SQLException { - return parse(rs.getString(columnName)); - } - - @Override - public T getNullableResult(ResultSet rs, int columnIndex) throws SQLException { - return parse(rs.getString(columnIndex)); - } - - @Override - public T getNullableResult(CallableStatement cs, int columnIndex) throws SQLException { - return parse(cs.getString(columnIndex)); - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/controller/LabEnvController.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/controller/LabEnvController.java deleted file mode 100644 index 5991761f2ad8ad3dbeb431c0ec2636c1ebc28afa..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/controller/LabEnvController.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.controller; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.service.LabEnvService; -import com.huawei.ic.openlab.cloudtest.service.TarFileService; -import com.huawei.ic.openlab.cloudtest.util.Constants; -import com.huawei.ic.openlab.cloudtest.util.NormalResp; - -import lombok.extern.slf4j.Slf4j; - - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RequestPart; -import org.springframework.web.bind.annotation.RestController; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; - -/** - * 实验室环境测试控制器类 - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -@RestController -@RequestMapping(value = "/api/v1", produces = MediaType.APPLICATION_JSON_VALUE) -public class LabEnvController { - private final TarFileService tarFileService; - private final LabEnvService labEnvService; - - - /** - * LabEnvController - * @param tarFileService labEnvService - * @param service LabEnvService - */ - @Autowired - public LabEnvController(TarFileService tarFileService, LabEnvService labEnvService) { - this.tarFileService = tarFileService; - this.labEnvService = labEnvService; - } - - /** - * 测试远程实验室连接是否正常 - * - * @param ip ip - * @param port port - * @param userName username - * @param passWord password - * @return response string - */ - @GetMapping(value = "/report") - public String setTestFile(@RequestParam(value = "file") String file, @RequestParam(value = "savePath") String savePath) throws IOException { - if (!Objects.requireNonNull(file).matches("^.*\\.tar.gz$")) { - throw new BaseException("文件后缀错误"); - } - labEnvService.resultJson(file, savePath); - return "success"; - } -} \ No newline at end of file diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/dao/CloudDevUseRecordDao.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/dao/CloudDevUseRecordDao.java deleted file mode 100644 index e44bd1951d8754a80bdf4af78eb08e01386bafee..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/dao/CloudDevUseRecordDao.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.dao; - -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; -import org.springframework.stereotype.Repository; - -/** - * 云开发平台访问记录 - * - * @since 2023-01-16 - * @author kongcaizhi - */ - -public interface CloudDevUseRecordDao { - /** - * Get record by taskId - * - * @param taskDeviceId task id - * @param kpToken kp token - * @return response string - */ - String getRecordById(@Param("taskDeviceId") String taskDeviceId, - @Param("kpToken") String kpToken); - - /** - * insert record - * - * @param requestTime requestTime - * @param taskDeviceId taskDeviceId - * @param kpToken KP TOKEN - * @return RESPONSE STRING - */ - int insertRecord(@Param("requestTime") String requestTime, - @Param("taskDeviceId") String taskDeviceId, - @Param("kpToken") String kpToken); -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/dao/CloudLabTestTaskDao.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/dao/CloudLabTestTaskDao.java deleted file mode 100644 index a398515d9777dafce7ecf2db8bc0ef7b8f019a03..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/dao/CloudLabTestTaskDao.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.dao; - -import com.huawei.ic.openlab.cloudtest.entity.CloudLabTestTask; -import com.huawei.ic.openlab.cloudtest.entity.LabTestReq; - -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; -import org.springframework.stereotype.Repository; - -import java.util.List; - -/** - * CloudLabTestTaskDao - * - * @author kongcaizhi - * @since 2021-12-15 - */ - -public interface CloudLabTestTaskDao { - /** - * get Test Task - * - * @param projectId projectId - * @return CloudLabTestTask - */ - CloudLabTestTask getTestTask(@Param("projectId") String projectId); - - /** - * get Ongoing Task - * - * @param data LabTestReq - * @return Task - */ - List getOngoingTask(@Param("data") LabTestReq data); - - /** - * set Test Task - * - * @param projectId projectId - * @param data data - */ - void setTestTask(@Param("projectId") String projectId, - @Param("data") CloudLabTestTask data); - - /** - * update Project Status - * - * @param projectId projectId - * @param status status - * @param startTime start Time - * @param finishTime finish Time - * @param statusDesc status Desc - * @return update num - */ - int updateProjectStatus(@Param("projectId") String projectId, - @Param("status") Integer status, - @Param("startTime") String startTime, - @Param("finishTime") String finishTime, - @Param("statusDesc") String statusDesc); - - /** - * update Upload File - * - * @param projectId projectId - * @param userId userId - * @param column column - * @param data data - * @param ip ip - * @return update num - */ - int updateUploadFile(@Param("projectId") String projectId, - @Param("userId") String userId, - @Param("column") String column, - @Param("data") CloudLabTestTask.UploadFile data, - @Param("ip") String ip); - - /** - * update Step Status - * - * @param data data - * @param testBeginTime test Begin Time - * @param projectId projectId - * @return update num - */ - int updateStepStatus(@Param("data") CloudLabTestTask.StepStatus data, - @Param("testBeginTime") CloudLabTestTask.TestBeginTime testBeginTime, - @Param("projectId") String projectId); - - /** - * update With Exception File - * - * @param data CloudLabTestTask - * @return update num - */ - int updateWithExceptionFile(@Param("data") CloudLabTestTask data); - - /** - * update Compatibility Result - * - * @param data CloudLabTestTask - * @return update num - */ - int updateCompatibilityResult(@Param("data") CloudLabTestTask data); - - /** - * update Performance Result - * - * @param data CloudLabTestTask - * @return update num - */ - int updatePerformanceResult(@Param("data") CloudLabTestTask data); -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/dao/PerformanceServiceDao.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/dao/PerformanceServiceDao.java deleted file mode 100644 index a54979c764e28b0afdf040c407587ab1bb618adf..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/dao/PerformanceServiceDao.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.dao; - -import com.huawei.ic.openlab.cloudtest.entity.PerformanceService; - -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; -import org.springframework.stereotype.Repository; - -import java.util.List; - -/** - * PerformanceServiceDao - * - * @author kongcaizhi - * @since 2021-12-15 - */ - -public interface PerformanceServiceDao { - /** - * Get Performance service - * - * @return get record - */ - List getPerformanceService(); - - /** - * Get Idle Service - * - * @return get record - */ - PerformanceService getIdleService(); - - /** - * add task - * - * @param serviceIp service ip - * @return add record - */ - int addTask(@Param("serviceIp") String serviceIp); - - /** - * Subtract task - * - * @param serviceIp service IP - * @return record - */ - int subtractTask(@Param("serviceIp") String serviceIp); -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/CloudLabTestTask.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/CloudLabTestTask.java deleted file mode 100644 index 351a79380c59affe08a12f8ae6e0900361f77ab5..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/CloudLabTestTask.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import org.apache.commons.lang3.StringUtils; - -import java.util.List; - -/** - * 远程实验室测试实体类 - * - * @author kongcaizhi - * @since 2021-11-11 - */ -@Data -public class CloudLabTestTask { - private String projectId; - private String userId; - private String serverIp; - private int serverPort; - private String serverUser; - private String serverPassword; - - private int projectStatus = 0; - private String statusDesc; - - private String requestTime; - private String startTime; - private String finishTime; - - @JsonProperty(value = "compatibilityTest") - private boolean isCompatibilityTest; - @JsonProperty(value = "reliabilityTest") - private boolean isReliabilityTest; - @JsonProperty(value = "securityTest") - private boolean isSecurityTest; - @JsonProperty(value = "functionTest") - private boolean isFunctionTest; - @JsonProperty(value = "performanceTest") - private boolean isPerformanceTest; - - private StepStatus stepStatus; - private String currentStatus; - private UploadFile functionFile; - private UploadFile performanceFile; - private UploadFile resultFile; - private TestCaseSummary testSummary; - private List testDetail; - private ScriptResultConfig scriptResultConfig; - private String performanceService; - private TestBeginTime testBeginTime; - private String taskLanguage; - - /** - * StepStatus - * - * @author kongcaizhi - * @since 2021-11-11 - */ - @AllArgsConstructor - @NoArgsConstructor - @Data - public static class StepStatus { - private String step1Code; - private String step1Time; - private UploadFile step1File; - private String step2Code; - private String step2Time; - private UploadFile step2File; - private String step3Code; - private String step3Time; - private UploadFile step3File; - private String step4Code; - private String step4Time; - private UploadFile step4File; - private String step5Code; - private String step5Time; - private UploadFile step5File; - private String step6Code; - private String step6Time; - private UploadFile step6File; - private String step7Code; - private String step7Time; - private UploadFile step7File; - private String step8Code; - private String step8Time; - private UploadFile step8File; - private String step9Code; - private String step9Time; - private UploadFile step9File; - private String step10Code; - private String step10Time; - private UploadFile step10File; - private String functionCode; - private String functionTime; - private UploadFile functionFile; - private String performanceCode; - private String performanceTime; - private UploadFile performanceFile; - - /** - * set Step1 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep1(String resultCode, String timeString) { - this.setStep1Code(resultCode); - this.setStep1Time(timeString); - } - - /** - * set Step2 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep2(String resultCode, String timeString) { - this.setStep2Code(resultCode); - this.setStep2Time(timeString); - } - - /** - * set Step3 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep3(String resultCode, String timeString) { - if (StringUtils.isNoneEmpty(this.getStep3Code())) { - this.setStep3Code(this.getStep3Code() + ";" + resultCode); - } else { - this.setStep3Code(resultCode); - } - this.setStep3Time(timeString); - } - - /** - * set Step4 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep4(String resultCode, String timeString) { - this.setStep4Code(resultCode); - this.setStep4Time(timeString); - } - - /** - * set Step5 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep5(String resultCode, String timeString) { - this.setStep5Code(resultCode); - this.setStep5Time(timeString); - } - - /** - * set Step6 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep6(String resultCode, String timeString) { - this.setStep6Code(resultCode); - this.setStep6Time(timeString); - } - - /** - * set Step7 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep7(String resultCode, String timeString) { - this.setStep7Code(resultCode); - this.setStep7Time(timeString); - } - - /** - * set Step8 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep8(String resultCode, String timeString) { - this.setStep8Code(resultCode); - this.setStep8Time(timeString); - } - - /** - * set Step9 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep9(String resultCode, String timeString) { - this.setStep9Code(resultCode); - this.setStep9Time(timeString); - } - - /** - * set Step10 - * - * @param resultCode resultCode - * @param timeString timeString - */ - public void setStep10(String resultCode, String timeString) { - this.setStep10Code(resultCode); - this.setStep10Time(timeString); - } - } - - /** - * UploadFile - * - * @author kongcaizhi - * @since 2021-11-11 - */ - @AllArgsConstructor - @NoArgsConstructor - @Data - public static class UploadFile { - private String fileDesc; - private String fileStatus; - private String fileId; - private String fileName; - private String filePath; - } - - /** - * TestBeginTime - * - * @author kongcaizhi - * @since 2021-11-11 - */ - @AllArgsConstructor - @NoArgsConstructor - @Data - public static class TestBeginTime { - private String compatibilityTime; - private String reliabilityTime; - private String securityTime; - private String functionTime; - private String performanceTime; - } - - /** - * TestCaseSummary - * - * @author kongcaizhi - * @since 2021-11-11 - */ - @AllArgsConstructor - @NoArgsConstructor - @Data - public static class TestCaseSummary { - private TestCaseResultCount function; - private TestCaseResultCount performance; - private TestCaseResultCount security; - private TestCaseResultCount compatibility; - private TestCaseResultCount reliability; - - public TestCaseSummary(TestCaseResultCount function, TestCaseResultCount security, - TestCaseResultCount compatibility, TestCaseResultCount reliability) { - this.function = function; - this.security = security; - this.compatibility = compatibility; - this.reliability = reliability; - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/LabTestReq.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/LabTestReq.java deleted file mode 100644 index a149413a4157d0e475319e20f46449c9722444b4..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/LabTestReq.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - -/** - * 网站请求蓝区兼容性测试实体类 - * - * @author kongcaizhi - * @since 2021-10-30 - */ -@Data -public class LabTestReq { - private String projectId; - private String userId; - - private String serverIp; - private int serverPort; - private String serverUser; - private String serverPassword; - - @JsonProperty(value = "compatibilityTest") - private boolean isCompatibilityTest; - @JsonProperty(value = "reliabilityTest") - private boolean isReliabilityTest; - @JsonProperty(value = "securityTest") - private boolean isSecurityTest; - @JsonProperty(value = "functionTest") - private boolean isFunctionTest; - @JsonProperty(value = "performance") - private boolean isPerformanceTest; - - private String deployDir; - private String applicationNames; - private String startAppCommands; - private String stopAppCommands; - - private int webPort; - - @JsonProperty(value = "language", defaultValue = "CN") - private String taskLanguage; -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/MqsMessage.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/MqsMessage.java deleted file mode 100644 index 35020a7b9ad818ccebde7bf3cb6504be8ea4111f..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/MqsMessage.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity; - -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * MQSMessage - * - * @author kongcaizhi - * @since 2021-10-30 - */ -@Data -public class MqsMessage { - private String projectId; - private String userId; - private String serverIp; - private String status; - private String statusDesc; - private String statusTime; - private MessageDetail detail; - private MessageResult testResult; - - /** - * MessageDetail - * - * @author kongcaizhi - * @since 2021-10-30 - */ - @AllArgsConstructor - @NoArgsConstructor - @Data - public static class MessageDetail { - private String step; - private String stepStatus; - private String exceptionType; - private String exceptionDesc; - private String exceptionFileName; - private String resultFileName; - private String fileStatus; - - /** - * DEPENDENCY_INSTALL_SUCCESS - */ - public void setDependencyInstallSuccess() { - this.setStep(Constants.DEPENDENCY_INSTALL_DESC); - this.setStepStatus(Constants.TEST_FINISHED_CN); - } - - /** - * APP_START_SUCCESS - */ - public void setAppStartSuccess() { - this.setStep(Constants.DEPENDENCY_INSTALL_DESC); - this.setStepStatus(Constants.TEST_FINISHED_CN); - } - - /** - * APP_STOP_SUCCESS - * - * @param isCompatibilityTest isCompatibilityTest - * @param isReliabilityTest isReliabilityTest - * @param isSecurityTest isSecurityTest - */ - public void setAppStopSuccess(boolean isCompatibilityTest, boolean isReliabilityTest, boolean isSecurityTest) { - this.setStep(Constants.APP_STOP_DESC); - this.setStepStatus(Constants.TEST_FINISHED_CN); - if (isCompatibilityTest) { - this.setStep(Constants.COMPATIBILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - if (isReliabilityTest) { - this.setStep(Constants.RELIABILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - if (isSecurityTest) { - this.setStep(Constants.SECURITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - } - - /** - * COM_START_SUCCESS - * - * @param isReliabilityTest isReliabilityTest - * @param isSecurityTest isSecurityTest - */ - public void setComStartSuccess(boolean isReliabilityTest, boolean isSecurityTest) { - this.setStep(Constants.COMPATIBILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - if (isReliabilityTest) { - this.setStep(Constants.RELIABILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - if (isSecurityTest) { - this.setStep(Constants.SECURITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - } - - /** - * PORT_TEST_SUCCESS - */ - public void setPortTestSuccess() { - this.setStep(Constants.DEPENDENCY_INSTALL_DESC); - this.setStepStatus(Constants.TEST_FINISHED_CN); - } - - /** - * VIRUS_SCAN_SUCCESS - * - * @param isCompatibilityTest isCompatibilityTest - * @param isReliabilityTest isReliabilityTest - */ - public void setVirusScanSuccess(boolean isCompatibilityTest, boolean isReliabilityTest) { - this.setStep(Constants.SECURITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - if (isCompatibilityTest) { - this.setStep(Constants.COMPATIBILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - if (isReliabilityTest) { - this.setStep(Constants.RELIABILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - } - - /** - * VULNERABLE_SCAN_SUCCESS - * - * @param isCompatibilityTest isCompatibilityTest - * @param isReliabilityTest isReliabilityTest - */ - public void setVulnerableScanSuccess(boolean isCompatibilityTest, boolean isReliabilityTest) { - this.setStep(Constants.SECURITY_TEST_DESC); - this.setStepStatus(Constants.TEST_FINISHED_CN); - if (isCompatibilityTest) { - this.setStep(Constants.COMPATIBILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - if (isReliabilityTest) { - this.setStep(Constants.RELIABILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - } - - /** - * EXCEPTION_TEST_FINISH - * - * @param isCompatibilityTest isCompatibilityTest - */ - public void setExceptionTestFinish(boolean isCompatibilityTest) { - this.setStep(Constants.RELIABILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_FINISHED_CN); - if (isCompatibilityTest) { - this.setStep(Constants.COMPATIBILITY_TEST_DESC); - this.setStepStatus(Constants.TEST_PROCESSING); - } - } - } - - /** - * MessageResult - * - * @author kongcaizhi - * @since 2021-10-30 - */ - @AllArgsConstructor - @NoArgsConstructor - @Data - public static class MessageResult { - private CloudLabTestTask.TestCaseSummary testSummary; - private List testDetail; - private ScriptResultConfig scriptResultConfig; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/PerformanceService.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/PerformanceService.java deleted file mode 100644 index c5d16022348a7b636cd352993ca67241cad2a860..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/PerformanceService.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity; - -import lombok.Data; - -/** - * PerformanceService - * - * @author kongcaizhi - * @since 2021-10-30 - */ -@Data -public class PerformanceService { - private String id; - private String deviceId; - private String serviceIp; - private String loginAccount; - private String loginPassword; - private String status; - private int taskNum; -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/ScriptResultConfig.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/ScriptResultConfig.java deleted file mode 100644 index 7ff6688a20a5939e83902133c76a24873d20cb9f..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/ScriptResultConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity; - -import lombok.Data; - -/** - * ScriptResultConfig - * - * @author kongcaizhi - * @since 2021-10-30 - */ -@Data -public class ScriptResultConfig { - private String applicationNames; - private String startAppCommands; - private String stopAppCommands; - private String osVersion; -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/SystemParams.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/SystemParams.java deleted file mode 100644 index 51f8b82039edcdccdc1c5a20f4067a34b5c4a3e8..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/SystemParams.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity; - -import lombok.Data; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * SystemParams - * - * @author kongcaizhi - * @since 2021-10-30 - */ -@Component -@Data -public class SystemParams { - @Value("${scriptConfigCn}") - private String scriptConfigCn; - - @Value("${scriptConfigEn}") - private String scriptConfigEn; - - @Value("${tempDir}") - private String tempDir; - - @Value("${deployIp}") - private String deployIP; - - @Value("${uploadFileDir}") - private String uploadFileDir; - - @Value("${appId}") - private String appId; - - @Value("${appKey}") - private String appKey; - - @Value("${mqsTopic}") - private String mqsTopic; - - @Value("${mqsUrl}") - private String mqsUrl; - - @Value("${clamdIp}") - private String clamdIp; - - @Value("${clamdPort}") - private int clamdPort; - - @Value("${performanceTestUrl}") - private String performanceTestUrl; - - - @Value("${appIdUat}") - private String appIdUat; - - @Value("${appKeyUat}") - private String appKeyUat; - - @Value("${mqsTopicUat}") - private String mqsTopicUat; - - @Value("${mqsUrlUat}") - private String mqsUrlUat; - - @Value("${kunpengServiceUrl}") - private String kunpengServiceUrl; - - /** - * get Script Config - * - * @param language language - * @return Config - */ - public String getScriptConfig(String language) { - return "CN".equals(language) ? scriptConfigCn : scriptConfigEn; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/TaskDelay.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/TaskDelay.java deleted file mode 100644 index 8ed1261353cbbf64231394c11ca37e08fd7b0583..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/TaskDelay.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity; - -import com.huawei.ic.openlab.cloudtest.util.Constants; -import com.huawei.ic.openlab.cloudtest.util.ToolUtil; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * TaskDelay - * - * @author kongcaizhi - * @since 2021-10-30 - */ -@Data -@Slf4j -@AllArgsConstructor -@NoArgsConstructor -public class TaskDelay { - private static final long PERFORMANCE_DELAY = 60 * 60 * 1000L; - private static final long DELAY = 60 * 60 * 1000L; - - private String projectId; - private Long startTime; - private Long updateTime; - private String currentStatus; - - /** - * 更新任务状态 - * - * @param currentStatus currentStatus - */ - public void updateCurrentStatus(String currentStatus) { - this.updateTime = ToolUtil.getMillionSeconds(); - this.currentStatus = currentStatus; - } - - /** - * 判断任务是否过期 - * - * @return isExpired - */ - public boolean isExpired() { - if (Constants.PERFORMANCE_TESTING.equals(currentStatus)) { - return (ToolUtil.getMillionSeconds() - this.updateTime) > PERFORMANCE_DELAY; - } else { - return (ToolUtil.getMillionSeconds() - this.updateTime) > DELAY; - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/TestCaseResult.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/TestCaseResult.java deleted file mode 100644 index e4889893196d7d946920a6d1c546d0261b83f0ad..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/TestCaseResult.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity; - -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; - -import lombok.Data; - -import org.springframework.util.StringUtils; - -import java.util.List; - -/** - * TaskDelay - * - * @author kongcaizhi - * @since 2021-10-30 - */ -@Data -public class TestCaseResult { - private String id; - private String result; - private String evidence; - private String reason; - - /** - * transferResult - * - * @return String - */ - public String transferResult() { - if (Constants.TEST_PASSED.equals(result)) { - return "通过"; - } else { - return "不通过"; - } - } - - /** - * transferEvidence - * - * @return String - */ - public String transferEvidence() { - JSONArray array = JSON.parseArray(evidence); - List list = array.toJavaList(String.class); - return String.join("", list); - } - - /** - * transferReason - * - * @return String - */ - public String transferReason() { - if (StringUtils.hasLength(reason)) { - return ",原因为：" + reason; - } else { - return ""; - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/TestCaseResultCount.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/TestCaseResultCount.java deleted file mode 100644 index 1f3a98490d4705f7763a1305f3ecf7f558a8b12c..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/TestCaseResultCount.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity; - -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * TestCaseResultCount - * - * @author kongcaizhi - * @since 2021-10-30 - */ -@AllArgsConstructor -@NoArgsConstructor -@Data -public class TestCaseResultCount { - private int total = 0; - private int passed = 0; - private int failed = 0; - private String startTime; - - /** - * add result - * - * @param result result - */ - public void add(String result) { - if (Constants.TEST_PASSED.equals(result)) { - passed += 1; - } else { - failed += 1; - } - total += 1; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/CompatibilityFilesParser.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/CompatibilityFilesParser.java deleted file mode 100644 index d4884e194c3f2cee0714bdcb8bb6e52cb2552766..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/CompatibilityFilesParser.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis; - -import java.io.IOException; - -/** - * CompatibilityFilesParser - * - * @author kongcaizhi - * @since 2021-12-15 - */ -public interface CompatibilityFilesParser { - /** - * Get result - * - * @return CompatibilityTestResult - * @throws IOException exception - */ - CompatibilityTestResult getResult() throws IOException; - - /** - * parseFiles - * - * @return CompatibilityTestResult - * @throws IOException exception - */ - CompatibilityTestResult parseFiles() throws IOException; -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/CompatibilityTestResult.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/CompatibilityTestResult.java deleted file mode 100644 index 56776b8f9ae6abf67c0d6ce1f3226b6957db7966..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/CompatibilityTestResult.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis; - -import lombok.Data; - -import java.util.ArrayList; -import java.util.List; - -/** - * CompatibilityTestResult - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Data -public class CompatibilityTestResult { - private String id; - private String result; - private List evidence = new ArrayList<>(); - private List evidenceEn = new ArrayList<>(); - private String reason; - private String reasonEn; - private String osVersion; - private List applicationNames; - private List startAppCommands; - private List stopAppCommands; - private List deployDir; - - /** - * CompatibilityTestResult - * - * @param testName test Name - */ - public CompatibilityTestResult(String testName) { - this.id = testName; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/ParseFunctionTestLog.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/ParseFunctionTestLog.java deleted file mode 100644 index 715878d1a7071bc7d9a507d3d34c9575032eccc2..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/ParseFunctionTestLog.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import com.alibaba.fastjson.JSONObject; - -import lombok.extern.slf4j.Slf4j; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * ParseFunctionTestLog - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseFunctionTestLog { - private static final Pattern RUN_PATTERN = Pattern.compile(Constants.PYTEST_RUN_PATTERN); - private static final Pattern SUCCESS_PATTERN = Pattern.compile(Constants.PYTEST_SUCCESS_PATTERN); - private static final Pattern FAIL_PATTERN = Pattern.compile(Constants.PYTEST_FAILED_PATTERN); - private static final Pattern SHELL_TEST_PATTERN = Pattern.compile(Constants.SHELL_UNIT_TEST_PATTERN); - private static final Pattern SHELL_RUN_PATTERN = Pattern.compile(Constants.SHELL_UNIT_RUN_PATTERN); - private static final Pattern SHELL_FAIL_PATTERN = Pattern.compile(Constants.SHELL_UNIT_FAILED_PATTERN); - - private final byte[] shellBytes; - private final byte[] pytestBytes; - - /** - * constructin function - * - * @param shellBytes shellBytes - * @param pytestBytes pytestBytes - */ - public ParseFunctionTestLog(byte[] shellBytes, byte[] pytestBytes) { - this.shellBytes = shellBytes; - this.pytestBytes = pytestBytes; - } - - /** - * get Shell Result - * - * @return CompatibilityTestResult - * @throws IOException IOException - */ - public CompatibilityTestResult getShellResult() { - Map resultMap = new HashMap<>(); - resultMap.put(Constants.TEST_FAILED, 0); - resultMap.put(Constants.TEST_PASSED, 0); - - List evidenceList = matches(resultMap); - CompatibilityTestResult testResult = new CompatibilityTestResult("Function_Test"); - testResult.setEvidence(evidenceList); - testResult.setReason(JSONObject.toJSONString(resultMap)); - return testResult; - } - - private List matches(Map resultMap) { - List evidenceList = new ArrayList<>(); - try (InputStream inputStream = new ByteArrayInputStream(shellBytes); - BufferedReader reader = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - String line; - int totalTest = 0; - while ((line = reader.readLine()) != null) { - match(line, evidenceList, resultMap, totalTest); - } - } catch (IOException e) { - throw new BaseException(e.getMessage()); - } - return evidenceList; - } - - private void match(String line, List evidenceList, Map resultMap, - int totalTest) { - int failTest; - int total = totalTest; - Matcher testMatch = SHELL_TEST_PATTERN.matcher(line); - if (testMatch.find()) { - evidenceList.add(line.replace(System.lineSeparator(), "")); - } - Matcher matcher = SHELL_RUN_PATTERN.matcher(line); - if (matcher.find()) { - evidenceList.add(line.replace(System.lineSeparator(), "")); - try { - total = Integer.parseInt(matcher.group(1).trim()); - } catch (NumberFormatException ex) { - total = 0; - } - } - if (line.matches("^\\\u001B\\[1;32mOK\\\u001B\\[0m")) { - evidenceList.add(line.replace(System.lineSeparator(), "")); - resultMap.put(Constants.TEST_PASSED, total + resultMap.get(Constants.TEST_PASSED)); - total = 0; - } - Matcher failedMatcher = SHELL_FAIL_PATTERN.matcher(line); - if (failedMatcher.find()) { - evidenceList.add(line.replace(System.lineSeparator(), "")); - try { - failTest = Integer.parseInt(failedMatcher.group(1).trim()); - } catch (NumberFormatException ex) { - failTest = 0; - } - if (failTest > total) { - failTest = total; - } - resultMap.put(Constants.TEST_FAILED, failTest + resultMap.get(Constants.TEST_FAILED)); - if (total != 0) { - resultMap.put(Constants.TEST_PASSED, - total - failTest + resultMap.get(Constants.TEST_PASSED)); - } - } - } - - private CompatibilityTestResult getPytestResult() { - List evidenceList = new ArrayList<>(); - Map resultMap = new HashMap<>(); - resultMap.put(Constants.TEST_FAILED, 0); - resultMap.put(Constants.TEST_PASSED, 0); - - try (InputStream inputStream = new ByteArrayInputStream(pytestBytes); - BufferedReader reader = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - String line; - while ((line = reader.readLine()) != null) { - matcher(line, evidenceList, resultMap); - } - } catch (IOException e) { - throw new BaseException(e.getMessage()); - } - CompatibilityTestResult testResult = new CompatibilityTestResult("Function_Test"); - testResult.setReason(JSONObject.toJSONString(resultMap)); - testResult.setEvidence(evidenceList); - return testResult; - } - - private void matcher(String line, List evidenceList, Map resultMap) { - Matcher runMatcher = RUN_PATTERN.matcher(line); - if (runMatcher.find()) { - evidenceList.add(line.replace(System.lineSeparator(), "")); - } - Matcher matcher = SUCCESS_PATTERN.matcher(line); - if (matcher.find()) { - evidenceList.add(line.replace(System.lineSeparator(), "")); - try { - resultMap.put(Constants.TEST_PASSED, - Integer.parseInt(matcher.group(1).trim()) + resultMap.get(Constants.TEST_PASSED)); - } catch (NumberFormatException ex) { - log.error("Error in parsing Python test results {}", line); - } - } - Matcher failedMatcher = FAIL_PATTERN.matcher(line); - if (failedMatcher.find()) { - evidenceList.add(line.replace(System.lineSeparator(), "")); - try { - resultMap.put(Constants.TEST_PASSED, - Integer.parseInt(failedMatcher.group(2).trim()) - + resultMap.get(Constants.TEST_PASSED)); - resultMap.put(Constants.TEST_FAILED, - Integer.parseInt(failedMatcher.group(1).trim()) - + resultMap.get(Constants.TEST_FAILED)); - } catch (NumberFormatException ex) { - log.error("Error in parsing Python test results {}", line); - } - } - } - - /** - * parseFunctionTestLog - * - * @return CompatibilityTestResult - */ - public CompatibilityTestResult parseFunctionTestLog() { - Map resultMap = new HashMap<>(); - List evidenceList = new ArrayList<>(); - CompatibilityTestResult testResult = new CompatibilityTestResult("Function_Test"); - try { - if (shellBytes.length != 0) { - CompatibilityTestResult shellResult = getShellResult(); - if (shellResult.getEvidence() != null && shellResult.getEvidence().size() > 0) { - evidenceList.addAll(shellResult.getEvidence()); - } - if (shellResult.getReason() != null) { - JSONObject jsonObject = JSONObject.parseObject(shellResult.getReason()); - resultMap.put(Constants.TEST_FAILED, jsonObject.getIntValue(Constants.TEST_FAILED)); - resultMap.put(Constants.TEST_PASSED, jsonObject.getIntValue(Constants.TEST_PASSED)); - } - } - if (pytestBytes.length != 0) { - CompatibilityTestResult pytestResult = getPytestResult(); - if (pytestResult.getEvidence() != null && pytestResult.getEvidence().size() > 0) { - evidenceList.addAll(pytestResult.getEvidence()); - } - if (pytestResult.getReason() != null) { - JSONObject jsonObject = JSONObject.parseObject(pytestResult.getReason()); - resultMap.put(Constants.TEST_FAILED, resultMap.getOrDefault(Constants.TEST_FAILED, 0) - + jsonObject.getIntValue(Constants.TEST_FAILED)); - resultMap.put(Constants.TEST_PASSED, resultMap.getOrDefault(Constants.TEST_PASSED, 0) - + jsonObject.getIntValue(Constants.TEST_PASSED)); - } - } - testResult.setResult(Constants.TEST_PASSED); - testResult.setReason(JSONObject.toJSONString(resultMap)); - testResult.setEvidence(evidenceList); - if (pytestBytes.length == 0 & shellBytes.length == 0) { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason("没有功能测试文件"); - } - } catch (BaseException e) { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason("功能测试文件解析出错"); - } - return testResult; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/ParseInfoLog.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/ParseInfoLog.java deleted file mode 100644 index 6a3e6a7c8b3b08d0ca93a823cca037799fd46053..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/ParseInfoLog.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis; - -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * ParseInfoLog - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseInfoLog { - private final byte[] infoLogBytes; - private String osVersion; - - /** - * construction function - * - * @param infoLogBytes info log bytes - * @param osVersion os version - */ - public ParseInfoLog(byte[] infoLogBytes, String osVersion) { - this.infoLogBytes = infoLogBytes; - this.osVersion = osVersion; - } - - private Map initMap() { - Map resultMap = new HashMap<>(); - resultMap.put(Constants.CompatibilityTestName.APPLICATION_START.getTestName(), - new CompatibilityTestResult(Constants.CompatibilityTestName.APPLICATION_START.getTestName())); - resultMap.put(Constants.CompatibilityTestName.APPLICATION_STOP.getTestName(), - new CompatibilityTestResult(Constants.CompatibilityTestName.APPLICATION_STOP.getTestName())); - resultMap.put(Constants.CompatibilityTestName.EXCEPTION_KILL.getTestName(), - new CompatibilityTestResult(Constants.CompatibilityTestName.EXCEPTION_KILL.getTestName())); - return resultMap; - } - - /** - * parse info log - * - * @return Map - */ - public Map parseInfoLog() { - Map resultMap = initMap(); - if (infoLogBytes.length == 0) { - for (Map.Entry entry : resultMap.entrySet()) { - CompatibilityTestResult testResult = entry.getValue(); - testResult.setResult(Constants.INFO_TEST_CASE_RESULT_MAP.get("FILE_NOT_EXIT").getTestResult()); - testResult.setReason(Constants.INFO_TEST_CASE_RESULT_MAP.get("FILE_NOT_EXIT").getTestReason()); - testResult.setReasonEn(Constants.INFO_TEST_CASE_RESULT_MAP.get("FILE_NOT_EXIT").getTestReasonEn()); - entry.setValue(testResult); - } - return resultMap; - } - try { - resultMap = readInfoLog(); - } catch (IllegalStateException | IOException ex) { - log.error("Exception in parsing info.log ", ex); - for (Map.Entry entry : resultMap.entrySet()) { - CompatibilityTestResult testResult = entry.getValue(); - testResult.setResult(Constants.INFO_TEST_CASE_RESULT_MAP.get("READ_ERROR").getTestResult()); - testResult.setReason(Constants.INFO_TEST_CASE_RESULT_MAP.get("READ_ERROR").getTestReason()); - testResult.setReasonEn(Constants.INFO_TEST_CASE_RESULT_MAP.get("READ_ERROR").getTestReasonEn()); - entry.setValue(testResult); - } - } catch (Exception ex) { - log.error("Exception in uncompressing info.log ", ex); - for (Map.Entry entry : resultMap.entrySet()) { - CompatibilityTestResult testResult = entry.getValue(); - testResult.setResult(Constants.INFO_TEST_CASE_RESULT_MAP.get("UNZIP_ERROR").getTestResult()); - testResult.setReason(Constants.INFO_TEST_CASE_RESULT_MAP.get("UNZIP_ERROR").getTestReason()); - testResult.setReasonEn(Constants.INFO_TEST_CASE_RESULT_MAP.get("UNZIP_ERROR").getTestReasonEn()); - entry.setValue(testResult); - } - } - return resultMap; - } - - /** - * get os version - * - * @return os version - */ - public String getOsVersion() { - return osVersion; - } - - /** - * 读取Info日志 - * - * @return map - * @throws IOException exception - */ - private Map readInfoLog() throws IOException { - Map> evidenceMap = new HashMap<>(); - - try (InputStream inputStream = new ByteArrayInputStream(infoLogBytes); - BufferedReader reader = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - List beginTestIndex = new ArrayList<>(); - String line; - int count = 0; - reader.mark(infoLogBytes.length + 1); - while ((line = reader.readLine()) != null) { - count++; - if (StringUtils.contains(line, Constants.START_TEST_STRING) || StringUtils.contains(line, - Constants.START_TEST_STRING_EN)) { - beginTestIndex.add(count); - } - } - int lastBeginTest = Collections.max(beginTestIndex); - reader.reset(); - - for (int i = 0; i <= lastBeginTest; i++) { - reader.readLine(); - } - - StringBuilder stringBuilder = new StringBuilder(16); - while ((line = reader.readLine()) != null) { - stringBuilder.append(line).append(System.lineSeparator()); - } - - Matcher osMatcher = Pattern.compile(Constants.OS_PATTERN_CN).matcher(stringBuilder.toString()); - if (osMatcher.find()) { - osVersion = osMatcher.group(3); - } - for (Map.Entry entry : Constants.LOG_WHITE_RULES.entrySet()) { - Pattern pattern = entry.getValue(); - Matcher matcher = pattern.matcher(stringBuilder.toString()); - if (matcher.find()) { - List evidenceList = evidenceMap.getOrDefault(entry.getKey(), new ArrayList<>()); - evidenceList.add(matcher.group(0).replace(System.lineSeparator(), "")); - evidenceMap.put(entry.getKey(), evidenceList); - } - } - } - return getResultMap(evidenceMap); - } - - /** - * 根据日志解析内容,返回用例结果. - * - * @param evidenceMap map - * @return map - */ - private Map getResultMap(Map> evidenceMap) { - CompatibilityTestResult startTest = - new CompatibilityTestResult(Constants.CompatibilityTestName.APPLICATION_START.getTestName()); - CompatibilityTestResult stopTest = - new CompatibilityTestResult(Constants.CompatibilityTestName.APPLICATION_STOP.getTestName()); - CompatibilityTestResult exceptionTest = - new CompatibilityTestResult(Constants.CompatibilityTestName.EXCEPTION_KILL.getTestName()); - startTest.setOsVersion(getOsVersion()); - - if (evidenceMap.size() > 0) { - for (Map.Entry> entry : evidenceMap.entrySet()) { - if (entry.getKey().contains("START_APP")) { - startTest.setResult(Constants.INFO_TEST_CASE_RESULT_MAP.get(entry.getKey()).getTestResult()); - startTest.setReason(Constants.INFO_TEST_CASE_RESULT_MAP.get(entry.getKey()).getTestReason()); - startTest.setReasonEn(Constants.INFO_TEST_CASE_RESULT_MAP.get(entry.getKey()).getTestReasonEn()); - startTest.setEvidence(entry.getValue()); - startTest.setEvidenceEn(entry.getValue()); - } - if (entry.getKey().contains("RELIABLE_TEST")) { - exceptionTest.setResult(Constants.INFO_TEST_CASE_RESULT_MAP.get(entry.getKey()).getTestResult()); - exceptionTest.setReason(Constants.INFO_TEST_CASE_RESULT_MAP.get(entry.getKey()).getTestReason()); - exceptionTest.setReasonEn( - Constants.INFO_TEST_CASE_RESULT_MAP.get(entry.getKey()).getTestReasonEn()); - exceptionTest.setEvidence(entry.getValue()); - exceptionTest.setEvidenceEn(entry.getValue()); - } - if (entry.getKey().contains("STOP_APP")) { - stopTest.setResult(Constants.INFO_TEST_CASE_RESULT_MAP.get(entry.getKey()).getTestResult()); - stopTest.setReason(Constants.INFO_TEST_CASE_RESULT_MAP.get(entry.getKey()).getTestReason()); - stopTest.setReasonEn(Constants.INFO_TEST_CASE_RESULT_MAP.get(entry.getKey()).getTestReasonEn()); - stopTest.setEvidence(entry.getValue()); - stopTest.setEvidenceEn(entry.getValue()); - } - } - } else { - startTest.setResult(Constants.INFO_TEST_CASE_RESULT_MAP.get("NO_START_APP").getTestResult()); - startTest.setReason(Constants.INFO_TEST_CASE_RESULT_MAP.get("NO_START_APP").getTestReason()); - startTest.setReasonEn(Constants.INFO_TEST_CASE_RESULT_MAP.get("NO_START_APP").getTestReasonEn()); - stopTest.setResult(Constants.INFO_TEST_CASE_RESULT_MAP.get("NO_STOP_APP").getTestResult()); - stopTest.setReason(Constants.INFO_TEST_CASE_RESULT_MAP.get("NO_STOP_APP").getTestReason()); - stopTest.setReasonEn(Constants.INFO_TEST_CASE_RESULT_MAP.get("NO_STOP_APP").getTestReasonEn()); - exceptionTest.setResult(Constants.INFO_TEST_CASE_RESULT_MAP.get("NO_RELIABLE_TEST").getTestResult()); - exceptionTest.setReason(Constants.INFO_TEST_CASE_RESULT_MAP.get("NO_RELIABLE_TEST").getTestReason()); - exceptionTest.setReasonEn(Constants.INFO_TEST_CASE_RESULT_MAP.get("NO_RELIABLE_TEST").getTestReasonEn()); - } - Map resultMap = new HashMap<>(); - resultMap.put(Constants.CompatibilityTestName.APPLICATION_START.getTestName(), startTest); - resultMap.put(Constants.CompatibilityTestName.APPLICATION_STOP.getTestName(), stopTest); - resultMap.put(Constants.CompatibilityTestName.EXCEPTION_KILL.getTestName(), exceptionTest); - return resultMap; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/ParseSafetyFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/ParseSafetyFiles.java deleted file mode 100644 index 74f7f628c295c497c967d8db3d8fe7a2807672eb..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/ParseSafetyFiles.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -/** - * ParseSafetyFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseSafetyFiles { - private final byte[] protocolBytes; - private final byte[] udpBytes; - private final byte[] tcpBytes; - - /** - * construction function - * - * @param protocolBytes bytes - * @param udpBytes udp bytes - * @param tcpBytes tcp bytes - */ - public ParseSafetyFiles(byte[] protocolBytes, byte[] udpBytes, byte[] tcpBytes) { - this.protocolBytes = protocolBytes; - this.udpBytes = udpBytes; - this.tcpBytes = tcpBytes; - } - - /** - * get average - * - * @param fileBytes file bytes - * @return average list - * @throws IOException exception - */ - public List getAverage(byte[] fileBytes) throws IOException { - List averageList = new ArrayList<>(); - try (InputStream beginInput = new ByteArrayInputStream(fileBytes); - BufferedReader beginReader = new BufferedReader( - new InputStreamReader(beginInput, StandardCharsets.UTF_8))) { - String line; - while ((line = beginReader.readLine()) != null) { - if (!StringUtils.startsWith(line, "#") - || StringUtils.containsIgnoreCase(line, "# Ports scanned")) { - averageList.add(line.trim()); - } - } - } - return averageList; - } - - /** - * read safety file - * - * @return CompatibilityTestResult - * @throws IOException exception - */ - public CompatibilityTestResult readSafetyFile() throws IOException { - if (protocolBytes.length == 0 || udpBytes.length == 0 || tcpBytes.length == 0) { - throw new BaseException("端口扫描文件缺失"); - } - List protocolList = getAverage(protocolBytes); - List tcpList = getAverage(tcpBytes); - List udpList = getAverage(udpBytes); - - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.SECURITY_PORT.getTestName()); - if (protocolList.size() > 0 && tcpList.size() > 0 && udpList.size() > 0) { - testResult.setResult(Constants.TEST_PASSED); - List evidenceList = new ArrayList<>(); - List evidenceEnList = new ArrayList<>(); - evidenceList.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(5), "PROTOCOL")); - evidenceList.addAll(protocolList); - evidenceList.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(5), "UDP")); - evidenceList.addAll(udpList); - evidenceList.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(5), "TCP")); - evidenceList.addAll(tcpList); - testResult.setEvidence(evidenceList); - evidenceEnList.add(String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_EN_LIST.get(5), "PROTOCOL")); - evidenceEnList.addAll(protocolList); - evidenceEnList.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(5), "UDP")); - evidenceEnList.addAll(udpList); - evidenceEnList.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(5), "TCP")); - evidenceEnList.addAll(tcpList); - testResult.setEvidenceEn(evidenceEnList); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SAFETY_FILE_LOSS); - testResult.setReasonEn(Constants.SAFETY_FILE_LOSS_EN); - } - return testResult; - } - - /** - * parseSafetyFile - * - * @return CompatibilityTestResult - */ - public CompatibilityTestResult parseSafetyFile() { - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.SECURITY_PORT.getTestName()); - try { - testResult = readSafetyFile(); - } catch (BaseException ex) { - log.error("Missing security file", ex.getMessage()); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SAFETY_FILE_LOSS); - testResult.setReasonEn(Constants.SAFETY_FILE_LOSS_EN); - } catch (NumberFormatException | IOException ex) { - log.error("Security file parsing error", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_LIST.get(2), "安全测试")); - testResult.setReasonEn(String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_EN_LIST.get(2), "security " - + "test")); - } - return testResult; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/AbsParseCompatibilityFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/AbsParseCompatibilityFiles.java deleted file mode 100644 index 2b4aaf3c570ecdae851266c82224203f8cd423b5..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/AbsParseCompatibilityFiles.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * AbsParseCompatibilityFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -abstract class AbsParseCompatibilityFiles { - private final Map beginByteMap; - private final Map endByteMap; - - /** - * AbsParseCompatibilityFiles init - * - * @param beginByteMap beginByteMap - * @param endByteMap endByteMap - */ - protected AbsParseCompatibilityFiles(Map beginByteMap, Map endByteMap) { - this.beginByteMap = beginByteMap; - this.endByteMap = endByteMap; - } - - /** - * read File - * - * @param beginBytes beginBytes - * @param endBytes endBytes - * @return CompatibilityTestResult - * @throws IOException Exception - */ - abstract CompatibilityTestResult readFile(byte[] beginBytes, byte[] endBytes) throws IOException; - - /** - * Get average rate - * - * @param fileBytes file bytes - * @return average list - * @throws IOException exception - */ - abstract List getAverage(byte[] fileBytes) throws IOException; - - /** - * Get Result - * - * @param testName Test name - * @param fileType file type - * @param fileTypeEn file type in English - * @param descIndex description index - * @return CompatibilityTestResult - * @throws IOException exception - */ - public CompatibilityTestResult getResult(String testName, String fileType, String fileTypeEn, int descIndex) - throws IOException { - if (beginByteMap == null || beginByteMap.size() == 0) { - throw new BaseException(String.format(Locale.ROOT, "解析空载测试%s文件缺失", fileType)); - } - if (endByteMap == null || endByteMap.size() == 0) { - throw new BaseException(String.format(Locale.ROOT, "解析空载测试%s文件缺失", fileType)); - } - List clusterResult = new ArrayList<>(); - List beginFileNames = beginByteMap.keySet().stream().filter(s -> StringUtils.endsWith(s, ".log")) - .collect(Collectors.toList()); - List endFileNames = endByteMap.keySet().stream().filter(s -> StringUtils.endsWith(s, ".log")) - .collect(Collectors.toList()); - - if (beginFileNames.size() > 0 && endFileNames.size() > 0) { - clusterResult.add(readFile(beginByteMap.get(beginFileNames.get(0)), endByteMap.get(endFileNames.get(0)))); - } - - if (clusterResult.size() > 0) { - if (clusterResult.stream().anyMatch(s -> StringUtils.contains(s.getResult(), Constants.TEST_FAILED))) { - return clusterResult.stream().filter(s -> StringUtils.contains(s.getResult(), Constants.TEST_FAILED)) - .collect(Collectors.toList()).get(0); - } else { - return clusterResult.get(0); - } - } - CompatibilityTestResult testResult = new CompatibilityTestResult(testName); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(descIndex), fileType)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(descIndex), - fileTypeEn)); - return testResult; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/AbsParsePerformanceFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/AbsParsePerformanceFiles.java deleted file mode 100644 index 35be92e5c7ba9754c7cc67ad8f03f421816857dd..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/AbsParsePerformanceFiles.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * AbsParsePerformanceFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -abstract class AbsParsePerformanceFiles { - private final Map inputByteMap; - - /** - * Construction function - * - * @param inputByteMap map - */ - public AbsParsePerformanceFiles(Map inputByteMap) { - this.inputByteMap = inputByteMap; - } - - /** - * Get result - * - * @param testName testName - * @param fileType file Type - * @param fileTypeEn file type in English - * @param descIndex description index - * @return CompatibilityTestResult object - * @throws IOException IOException - */ - public CompatibilityTestResult getResult(String testName, String fileType, String fileTypeEn, int descIndex) - throws IOException { - CompatibilityTestResult testResult = new CompatibilityTestResult(testName); - if (inputByteMap == null || inputByteMap.size() == 0) { - throw new BaseException(String.format(Locale.ROOT, "解析压力测试%s文件缺失", fileType)); - } - List beginFileNames = inputByteMap.keySet().stream().filter(s -> StringUtils.endsWith(s, ".log")) - .collect(Collectors.toList()); - if (beginFileNames.size() > 0) { - return readFile(inputByteMap.get(beginFileNames.get(0))); - } - - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(descIndex), fileType)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(descIndex), - fileTypeEn)); - return testResult; - } - - /** - * read file - * - * @param fileBytes file bytes - * @return CompatibilityTestResult - * @throws IOException Exception - */ - abstract CompatibilityTestResult readFile(byte[] fileBytes) throws IOException; - - /** - * Get Idle list - * - * @param fileBytes FILE BYTES - * @return IDLE List - * @throws IOException exception - */ - abstract List getIdleList(byte[] fileBytes) throws IOException; - - /** - * GET idle map - * - * @param fileBytes fileBytes - * @return idle map - * @throws IOException exception - */ - abstract Map>> getIdleMap(byte[] fileBytes) throws IOException; -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseClamLog.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseClamLog.java deleted file mode 100644 index f823c8b9a84128a5a1ed47cd904f4da1c78ecf2d..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseClamLog.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * AbsParsePerformanceFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseClamLog implements CompatibilityFilesParser { - private static final String RESULT_DESC = "result"; - private static final String EVIDENCE_DESC = "evidence"; - - private final Map inputByteMap; - - /** - * Construction function - * - * @param inputByteMap map - */ - public ParseClamLog(Map inputByteMap) { - this.inputByteMap = inputByteMap; - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.SECURITY_VIRUS.getTestName()); - if (inputByteMap == null || inputByteMap.size() == 0) { - throw new BaseException("病毒扫描文件缺失"); - } - List beginFileNames = inputByteMap.keySet().stream().filter(s -> StringUtils.contains(s, - "clam.log")) - .collect(Collectors.toList()); - if (beginFileNames.size() > 0) { - return readVirusScanFile(inputByteMap.get(beginFileNames.get(0))); - } - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SAFETY_FILE_LOSS); - testResult.setReasonEn(Constants.SAFETY_FILE_LOSS_EN); - return testResult; - } - - private Map> getScanResult(BufferedReader reader) throws IOException { - String result = null; - List evidenceList = new ArrayList<>(); - String line; - while ((line = reader.readLine()) != null) { - String lineTrim = line.trim(); - evidenceList.add(lineTrim); - if (lineTrim.isEmpty()) { - break; - } - Pattern pattern = Pattern.compile(Constants.CLAM_RESULT_PATTERN); - Matcher matcher = pattern.matcher(line); - if (matcher.find()) { - result = matcher.group(1).trim(); - } - } - Map> resultMap = new HashMap<>(); - resultMap.put(RESULT_DESC, Collections.singletonList(result)); - resultMap.put(EVIDENCE_DESC, evidenceList); - return resultMap; - } - - /** - * Read virus scan file - * - * @param clamFileBytes clam file bytes - * @return CompatibilityTestResult - * @throws IOException exception - */ - public CompatibilityTestResult readVirusScanFile(byte[] clamFileBytes) throws IOException { - if (clamFileBytes.length == 0) { - throw new BaseException("病毒扫描文件的内容为空"); - } - - Map> resultMap = new HashMap<>(); - try (InputStream inputStream = new ByteArrayInputStream(clamFileBytes); - BufferedReader bufferedReader = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - String line; - while ((line = bufferedReader.readLine()) != null) { - if (StringUtils.containsIgnoreCase(line, "SCAN SUMMARY")) { - resultMap = getScanResult(bufferedReader); - } - } - } - - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.SECURITY_VIRUS.getTestName()); - if (resultMap.containsKey(RESULT_DESC) && !CollectionUtils.isEmpty(resultMap.get(RESULT_DESC))) { - if (StringUtils.equals(resultMap.get(RESULT_DESC).get(0), "0")) { - testResult.setResult(Constants.TEST_PASSED); - } else { - testResult.setResult(Constants.TEST_FAILED); - } - if (resultMap.containsKey(EVIDENCE_DESC) && resultMap.get(EVIDENCE_DESC) != null) { - testResult.setEvidence(resultMap.get(EVIDENCE_DESC)); - testResult.setEvidenceEn(resultMap.get(EVIDENCE_DESC)); - } - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SAFETY_FILE_LOSS); - testResult.setReasonEn(Constants.SAFETY_FILE_LOSS_EN); - } - return testResult; - } - - @Override - public CompatibilityTestResult parseFiles() { - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.SECURITY_VIRUS.getTestName()); - try { - testResult = getResult(); - } catch (IOException ex) { - log.error("Exception occur in parsing virus file", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SAFETY_FILE_LOSS); - } - return testResult; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseCpuComFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseCpuComFiles.java deleted file mode 100644 index 129b1f031da3e236053f2fb48ef698b396d0a3f0..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseCpuComFiles.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -/** - * ParseCpuComFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseCpuComFiles implements CompatibilityFilesParser { - private static final String CPU_DESC = "CPU"; - - private final ParseCompatibilityFilesImpl parseCompatibilityFiles; - - /** - * ParseCpuComFiles - * - * @param beginCpuByteMap beginCpuByteMap - * @param endCpuByteMap endCpuByteMap - */ - public ParseCpuComFiles(Map beginCpuByteMap, Map endCpuByteMap) { - this.parseCompatibilityFiles = new ParseCompatibilityFilesImpl(beginCpuByteMap, endCpuByteMap); - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - return parseCompatibilityFiles.getResult(Constants.CompatibilityTestName.IDLE_CPU.getTestName(), - CPU_DESC, CPU_DESC, 2); - } - - @Override - public CompatibilityTestResult parseFiles() { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.IDLE_CPU.getTestName()); - try { - testResult = getResult(); - } catch (BaseException ex) { - log.error("Missing idle test cpu file ", ex.getMessage()); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(3), CPU_DESC)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(3), CPU_DESC)); - } catch (NumberFormatException | IOException ex) { - log.error("Exception in parsing idle test cpu file ", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(2), CPU_DESC)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(2), CPU_DESC)); - } - return testResult; - } - - private static class ParseCompatibilityFilesImpl extends AbsParseCompatibilityFiles { - /** - * ParseCompatibilityFilesImpl - * - * @param beginByteMap beginByteMap - * @param endByteMap endByteMap - */ - protected ParseCompatibilityFilesImpl(Map beginByteMap, Map endByteMap) { - super(beginByteMap, endByteMap); - } - - @Override - CompatibilityTestResult readFile(byte[] beginBytes, byte[] endBytes) throws IOException { - if (beginBytes.length == 0 || endBytes.length == 0) { - throw new BaseException("解析空载测试CPU文件缺失"); - } - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.IDLE_CPU.getTestName()); - List beginAverageList = getAverage(beginBytes); - List endAverageList = getAverage(endBytes); - double cpuMaxUsedIdle; - if (beginAverageList.size() > 0 && endAverageList.size() > 0) { - cpuMaxUsedIdle = - new BigDecimal(beginAverageList.get(0)) - .subtract(new BigDecimal(endAverageList.get(0))).abs().doubleValue(); - if (cpuMaxUsedIdle <= 1.0) { - testResult.setResult(Constants.TEST_PASSED); - List evidence = Arrays.asList(beginAverageList.get(1), endAverageList.get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(1), CPU_DESC, - cpuMaxUsedIdle, "小于")); - List evidenceEn = Arrays.asList(beginAverageList.get(1), endAverageList.get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(1), - CPU_DESC, cpuMaxUsedIdle, "less")); - testResult.setEvidence(evidence); - testResult.setEvidenceEn(evidenceEn); - } else { - testResult.setResult(Constants.TEST_FAILED); - List evidence = Arrays.asList(beginAverageList.get(1), endAverageList.get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(1), CPU_DESC, - cpuMaxUsedIdle, "大于")); - testResult.setEvidence(evidence); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(1), - CPU_DESC, cpuMaxUsedIdle, "大于")); - List evidenceEn = Arrays.asList(beginAverageList.get(1), endAverageList.get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(1), - CPU_DESC, cpuMaxUsedIdle, "greater")); - testResult.setEvidenceEn(evidenceEn); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(1), - CPU_DESC, cpuMaxUsedIdle, "greater")); - } - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(2), CPU_DESC)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(2), - CPU_DESC)); - } - return testResult; - } - - @Override - List getAverage(byte[] fileBytes) throws IOException { - List averageList = new ArrayList<>(); - try (InputStream beginInput = new ByteArrayInputStream(fileBytes); - BufferedReader beginReader = new BufferedReader( - new InputStreamReader(beginInput, StandardCharsets.UTF_8))) { - String line; - - while ((line = beginReader.readLine()) != null) { - if (StringUtils.containsIgnoreCase(line, "平均时间") - || StringUtils.startsWithIgnoreCase(line, "Average")) { - averageList.add(line.substring(line.length() - 8).trim()); - averageList.add(line.replace(System.lineSeparator(), "")); - } - } - } - return averageList; - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseCpuPerFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseCpuPerFiles.java deleted file mode 100644 index b503e9ef7904b612cec87a17cc011e710c3df674..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseCpuPerFiles.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Pattern; - -/** - * ParseCpuPerFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseCpuPerFiles implements CompatibilityFilesParser { - private static final String CPU_DESC = "CPU"; - - private final ParsePerformanceFilesImpl parsePerformanceFiles; - - /** - * construction function - * - * @param inputByteMap map - */ - public ParseCpuPerFiles(Map inputByteMap) { - this.parsePerformanceFiles = new ParsePerformanceFilesImpl(inputByteMap); - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - return parsePerformanceFiles.getResult(Constants.CompatibilityTestName.PRESSURE_CPU.getTestName(), - CPU_DESC, CPU_DESC, 2); - } - - @Override - public CompatibilityTestResult parseFiles() { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.PRESSURE_CPU.getTestName()); - try { - testResult = getResult(); - } catch (BaseException ex) { - log.error("Missing pressure test CPU file ", ex); - testResult.setResult(Constants.TEST_SKIPPED); - testResult.setReason(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(3), CPU_DESC)); - testResult.setReasonEn(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(3), CPU_DESC)); - } catch (NumberFormatException | IOException ex) { - log.error("Exception in parsing pressure test CPU file ", ex); - - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(2), CPU_DESC)); - testResult.setReasonEn(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(2), CPU_DESC)); - } - return testResult; - } - - private static class ParsePerformanceFilesImpl extends AbsParsePerformanceFiles { - private final Pattern idlePattern = Pattern.compile( - "\\d{2}(:|时)\\d{2}(:|分)\\d{2}(秒|\\s(PM|AM|HKT|))\\s+all\\s+"); - - /** - * ParsePerformanceFilesImpl - * - * @param inputByteMap inputByteMap - */ - public ParsePerformanceFilesImpl(Map inputByteMap) { - super(inputByteMap); - } - - @Override - CompatibilityTestResult readFile(byte[] fileBytes) throws IOException { - if (fileBytes.length == 0) { - throw new BaseException("解析压力测试CPU文件缺失"); - } - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.PRESSURE_CPU.getTestName()); - List idleList = getIdleList(fileBytes); - if (idleList.size() > 0) { - List evidence = new ArrayList<>(idleList.subList(1, idleList.size())); - List evidenceEn = new ArrayList<>(idleList.subList(1, idleList.size())); - if (Double.parseDouble(idleList.get(0)) <= 5.0d) { - testResult.setResult(Constants.TEST_PASSED); - evidence.add(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(0), CPU_DESC, - Double.parseDouble(idleList.get(0)), "小于")); - evidenceEn.add(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(0), CPU_DESC, - Double.parseDouble(idleList.get(0)), "less")); - } else { - testResult.setResult(Constants.TEST_FAILED); - evidence.add(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(0), CPU_DESC, - Double.parseDouble(idleList.get(0)), "大于")); - evidenceEn.add(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(0), CPU_DESC, - Double.parseDouble(idleList.get(0)), "greater")); - testResult.setReason(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(0), CPU_DESC, - Double.parseDouble(idleList.get(0)), "大于") - + Constants.PERFORMANCE_DESC_LIST.get(1)); - testResult.setReasonEn(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(0), - CPU_DESC, - Double.parseDouble(idleList.get(0)), "greater") - + Constants.PERFORMANCE_DESC_EN_LIST.get(1)); - } - testResult.setEvidence(evidence); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(2), CPU_DESC)); - testResult.setReasonEn(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(2), CPU_DESC)); - } - return testResult; - } - - @Override - List getIdleList(byte[] fileBytes) throws IOException { - Double maxIdle = 0.0d; - String maxEvidence = null; - Double minIdle = 100.0d; - String minEvidence = null; - try (InputStream beginInput = new ByteArrayInputStream(fileBytes); - BufferedReader beginReader = new BufferedReader( - new InputStreamReader(beginInput, StandardCharsets.UTF_8))) { - String line; - while ((line = beginReader.readLine()) != null) { - if (!idlePattern.matcher(line).find()){ - continue; - } - String idle = line.substring(line.length() - 8).trim(); - if (maxIdle <= Double.parseDouble(idle)) { - maxIdle = Double.parseDouble(idle); - maxEvidence = line; - } - if (minIdle >= Double.parseDouble(idle)) { - minIdle = Double.parseDouble(idle); - minEvidence = line; - } - } - } - - List idleList = new ArrayList<>(); - if (maxEvidence != null) { - idleList.add(String.format(Locale.ROOT, "%.2f", new BigDecimal(maxIdle.toString()) - .subtract(new BigDecimal(minIdle.toString())).abs().doubleValue())); - idleList.add(maxEvidence); - idleList.add(minEvidence); - } - return idleList; - } - - @Override - Map>> getIdleMap(byte[] fileBytes) { - return new HashMap<>(); - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseCveCheckFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseCveCheckFiles.java deleted file mode 100644 index 5cff564b93a7202ef19e80ad657b2c2421e09eed..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseCveCheckFiles.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * ParseCveCheckFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseCveCheckFiles implements CompatibilityFilesParser { - private final Map inputByteMap; - - /** - * construction function - * - * @param inputByteMap map - */ - public ParseCveCheckFiles(Map inputByteMap) { - this.inputByteMap = inputByteMap; - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.SECURITY_VULNERABLE.getTestName()); - if (inputByteMap == null || inputByteMap.size() == 0) { - throw new BaseException("漏洞扫描文件缺失"); - } - List beginFileNames = inputByteMap.keySet().stream().filter(s -> StringUtils.contains(s, - "cvecheck-result.json")) - .collect(Collectors.toList()); - if (beginFileNames.size() > 0) { - return readVirusScanFile(inputByteMap.get(beginFileNames.get(0))); - } - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SAFETY_FILE_LOSS); - testResult.setReasonEn(Constants.SAFETY_FILE_LOSS_EN); - return testResult; - } - - /** - * readVirusScanFile - * - * @param cveFileBytes cveFileBytes - * @return CompatibilityTestResult - * @throws IOException IOException - */ - public CompatibilityTestResult readVirusScanFile(byte[] cveFileBytes) throws IOException { - if (cveFileBytes.length == 0) { - throw new BaseException("漏洞扫描文件缺失"); - } - String line; - StringBuilder stringBuilder = new StringBuilder(16); - try (InputStream inputStream = new ByteArrayInputStream(cveFileBytes); - BufferedReader reader = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - while ((line = reader.readLine()) != null) { - stringBuilder.append(line); - } - } - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.SECURITY_VULNERABLE.getTestName()); - if (stringBuilder.toString().matches("(\\[\\])+")) { - testResult.setResult(Constants.TEST_PASSED); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setEvidence(Collections.singletonList(stringBuilder.toString())); - testResult.setEvidenceEn(Collections.singletonList(stringBuilder.toString())); - testResult.setReason(stringBuilder.toString()); - testResult.setReasonEn(stringBuilder.toString()); - } - return testResult; - } - - @Override - public CompatibilityTestResult parseFiles() throws IOException { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.SECURITY_VULNERABLE.getTestName()); - try { - testResult = getResult(); - } catch (NumberFormatException | IOException ex) { - log.error("Exception in parsing cve check file ", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SAFETY_FILE_LOSS); - testResult.setReasonEn(Constants.SAFETY_FILE_LOSS_EN); - } - return testResult; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseDiskComFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseDiskComFiles.java deleted file mode 100644 index 2aa2d925a436632b98290add35aaf9c65401b0dd..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseDiskComFiles.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Pattern; - -import org.apache.commons.lang3.StringUtils; -/** - * ParseDiskComFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseDiskComFiles implements CompatibilityFilesParser { - private static final String DRIVE_CN = "硬盘"; - private static final String DRIVE_EN = "drive"; - - private final ParseCompatibilityFilesImpl parseCompatibilityFiles; - - /** - * ParseDiskComFiles - * - * @param beginDiskByteMap beginDiskByteMap - * @param endDiskByteMap endDiskByteMap - */ - public ParseDiskComFiles(Map beginDiskByteMap, Map endDiskByteMap) { - this.parseCompatibilityFiles = new ParseCompatibilityFilesImpl(beginDiskByteMap, endDiskByteMap); - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - return parseCompatibilityFiles.getResult(Constants.CompatibilityTestName.IDLE_DISK.getTestName(), - DRIVE_CN, DRIVE_EN, 2); - } - - @Override - public CompatibilityTestResult parseFiles() { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.IDLE_DISK.getTestName()); - try { - testResult = getResult(); - } catch (BaseException ex) { - log.error("Missing idle test disk file ", ex.getMessage()); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(3), DRIVE_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(3), DRIVE_EN)); - } catch (NumberFormatException | IOException ex) { - log.error("Exception in parsing idle test disk file ", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(2), DRIVE_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(2), DRIVE_EN)); - } - return testResult; - } - - private static class ParseCompatibilityFilesImpl extends AbsParseCompatibilityFiles { - private final Pattern averagePattern = Pattern.compile("(平均时间|Average):"); - - /** - * ParseCompatibilityFilesImpl - * - * @param beginByteMap beginByteMap - * @param endByteMap endByteMap - */ - protected ParseCompatibilityFilesImpl(Map beginByteMap, Map endByteMap) { - super(beginByteMap, endByteMap); - } - - @Override - CompatibilityTestResult readFile(byte[] beginBytes, byte[] endBytes) throws IOException { - if (beginBytes.length == 0 || endBytes.length == 0) { - throw new BaseException("解析空载测试硬盘文件缺失"); - } - - Map> beginAverageMap = getAverageMap(beginBytes); - Map> endAverageMap = getAverageMap(endBytes); - - double diskMaxUsedIdle = 0.0d; - String maxDiskName = null; - double diskIdle = 0.0d; - if (beginAverageMap.size() > 0 && endAverageMap.size() > 0) { - for (Map.Entry> entry : beginAverageMap.entrySet()) { - if (!endAverageMap.containsKey(entry.getKey())) { - continue; - } - diskIdle = new BigDecimal(beginAverageMap.get(entry.getKey()).get(0)) - .subtract(new BigDecimal(endAverageMap.get(entry.getKey()).get(0))) - .abs().doubleValue(); - if (diskMaxUsedIdle <= diskIdle) { - maxDiskName = entry.getKey(); - diskMaxUsedIdle = diskIdle; - } - } - } - return getTestResult(maxDiskName, diskMaxUsedIdle, beginAverageMap, endAverageMap); - } - - private CompatibilityTestResult getTestResult(String maxDiskName, double diskMaxUsedIdle, - Map> beginAverageMap, Map> endAverageMap) { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.IDLE_DISK.getTestName()); - List evidence; - List evidenceEn; - if (maxDiskName != null) { - if (diskMaxUsedIdle <= 1.0) { - testResult.setResult(Constants.TEST_PASSED); - evidence = Arrays.asList(beginAverageMap.get(maxDiskName).get(1), - endAverageMap.get(maxDiskName).get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(1), DRIVE_CN, - diskMaxUsedIdle, "小于")); - evidenceEn = Arrays.asList(beginAverageMap.get(maxDiskName).get(1), - endAverageMap.get(maxDiskName).get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(1), - DRIVE_EN, diskMaxUsedIdle, "less")); - } else { - testResult.setResult(Constants.TEST_FAILED); - evidence = Arrays.asList(beginAverageMap.get(maxDiskName).get(1), - endAverageMap.get(maxDiskName).get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(1), - DRIVE_CN, diskMaxUsedIdle, "大于")); - evidenceEn = Arrays.asList(beginAverageMap.get(maxDiskName).get(1), - endAverageMap.get(maxDiskName).get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(1), - DRIVE_EN, diskMaxUsedIdle, "greater")); - } - testResult.setEvidence(evidence); - testResult.setEvidenceEn(evidenceEn); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(2), DRIVE_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(2), - DRIVE_EN)); - } - return testResult; - } - - @Override - List getAverage(byte[] fileBytes) { - return new ArrayList<>(); - } - - /** - * Get average map - * - * @param cpuBytes cpu bytes - * @return average map - * @throws IOException exception - */ - public Map> getAverageMap(byte[] cpuBytes) throws IOException { - String line = null; - Map> averageMap = new HashMap<>(); - Map indexMap = new HashMap<>(); - try (InputStream beginInput = new ByteArrayInputStream(cpuBytes); - BufferedReader beginReader = new BufferedReader( - new InputStreamReader(beginInput, StandardCharsets.UTF_8))) { - boolean flag = true; - while ((line = beginReader.readLine()) != null) { - if (!StringUtils.containsIgnoreCase(line, "DEV") && averagePattern.matcher(line).find()) { - String[] split = line.split("\\s+"); - String idle = split[indexMap.get("%util")]; - String diskName = split[indexMap.get("DEV")]; - List averageList = new ArrayList<>(); - averageList.add(idle); - averageList.add(line.replace("\n", "")); - averageMap.put(diskName, averageList); - } else if (flag && StringUtils.containsIgnoreCase(line, "DEV") - && averagePattern.matcher(line).find() ){ - indexMap = getIndexMap(line); - flag = false; - } - } - } - return averageMap; - } - - private static Map getIndexMap(String line) { - Map indexMap = new HashMap<>(); - String[] split =line.split("\\s+"); - for (int i = 0; i < split.length; i++) { - if (split[i].equals("DEV")) { - indexMap.put("DEV", i); - } else if (split[i].equals("%util")) { - indexMap.put("%util", i); - } - } - return indexMap; - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseDiskPerFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseDiskPerFiles.java deleted file mode 100644 index 7855b83070fbe1c2cc796936ee934ae72fb39ceb..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseDiskPerFiles.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * ParseDiskPerFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseDiskPerFiles implements CompatibilityFilesParser { - private static final String DRIVE_CN = "硬盘"; - private static final String DRIVE_EN = "drive"; - - private final ParsePerformanceFilesImpl parsePerformanceFiles; - - /** - * construction function - * - * @param inputByteMap MAP - */ - public ParseDiskPerFiles(Map inputByteMap) { - this.parsePerformanceFiles = new ParsePerformanceFilesImpl(inputByteMap); - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - return parsePerformanceFiles.getResult(Constants.CompatibilityTestName.PRESSURE_DISK.getTestName(), - DRIVE_CN, DRIVE_EN, 2); - } - - @Override - public CompatibilityTestResult parseFiles() { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.PRESSURE_DISK.getTestName()); - try { - testResult = getResult(); - } catch (BaseException ex) { - log.error("Missing pressure test disk file ", ex.getMessage()); - testResult.setResult(Constants.TEST_SKIPPED); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(3), DRIVE_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(3), DRIVE_EN)); - } catch (NumberFormatException | IOException ex) { - log.error("Exception occur in parsing pressure test disk file ", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(2), DRIVE_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(2), DRIVE_EN)); - } - return testResult; - } - - private static class ParsePerformanceFilesImpl extends AbsParsePerformanceFiles { - private final Pattern idlePattern = Pattern.compile( - "\\d{2}(:|时)\\d{2}(:|分)\\d{2}(秒|\\s(PM|AM|HKT|))\\s+"); - - /** - * ParsePerformanceFilesImpl - * - * @param inputByteMap inputByteMap - */ - public ParsePerformanceFilesImpl(Map inputByteMap) { - super(inputByteMap); - } - - @Override - CompatibilityTestResult readFile(byte[] fileBytes) throws IOException { - if (fileBytes.length == 0) { - throw new BaseException("解析压力测试硬盘文件缺失"); - } - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.PRESSURE_DISK.getTestName()); - Map>> idleMap = getIdleMap(fileBytes); - if (idleMap.size() > 0) { - Double maxGap = 0.00d; - String maxDiskName = null; - String diskMax = null; - String diskMin = null; - for (Map.Entry>> entry : idleMap.entrySet()) { - List idleList = entry.getValue().get("idle"); - double diskMaxGap = idleList.stream().mapToDouble(Double::parseDouble).max().orElse(0.00d); - double diskMinGap = idleList.stream().mapToDouble(Double::parseDouble).min().orElse(0.00d); - Double gap = new BigDecimal(String.valueOf(diskMaxGap)) - .subtract(new BigDecimal(String.valueOf(diskMinGap))).abs().doubleValue(); - if (maxGap <= gap) { - maxGap = gap; - maxDiskName = entry.getKey(); - diskMax = String.format(Locale.ROOT, "%.2f", diskMaxGap); - diskMin = String.format(Locale.ROOT, "%.2f", diskMinGap); - } - } - testResult = getTestResult(maxDiskName, maxGap, idleMap, diskMax, diskMin); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_LIST.get(2), DRIVE_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(2), DRIVE_EN)); - } - return testResult; - } - - private CompatibilityTestResult getTestResult(String maxDiskName, Double maxGap, - Map>> idleMap, - String diskMax, String diskMin) { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.PRESSURE_DISK.getTestName()); - List evidenceList = idleMap.get(maxDiskName).get("evidence"); - List evidence = new ArrayList<>(); - evidence.add(evidenceList.stream().filter( - s -> StringUtils.containsIgnoreCase(s, diskMax)).limit(1) - .collect(Collectors.joining())); - evidence.add(evidenceList.stream().filter( - s -> StringUtils.containsIgnoreCase(s, diskMin)).limit(1) - .collect(Collectors.joining())); - List evidenceEn = new ArrayList<>(evidence); - if (maxGap <= 5.0) { - testResult.setResult(Constants.TEST_PASSED); - evidence.add(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(0), DRIVE_CN, maxGap, - "小于")); - evidenceEn.add(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(0), DRIVE_EN, maxGap, "less")); - } else { - testResult.setResult(Constants.TEST_FAILED); - evidence.add(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(0), - DRIVE_CN, maxGap, "大于")); - evidenceEn.add(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(0), - DRIVE_EN, maxGap, "greater")); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(0), - DRIVE_CN, maxGap, "大于")); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(0), - DRIVE_EN, maxGap, "greater")); - } - testResult.setEvidence(evidence); - testResult.setEvidenceEn(evidenceEn); - return testResult; - } - - @Override - List getIdleList(byte[] fileBytes) { - return new ArrayList<>(); - } - - @Override - Map>> getIdleMap(byte[] fileBytes) throws IOException { - String line = null; - Map>> averageMap = new HashMap<>(); - Map indexMap = new HashMap<>(); - try (InputStream beginInput = new ByteArrayInputStream(fileBytes); - BufferedReader beginReader = new BufferedReader(new InputStreamReader(beginInput, - StandardCharsets.UTF_8))) { - boolean flag = true; - - while ((line = beginReader.readLine()) != null) { - if (!StringUtils.containsIgnoreCase(line, "DEV") && idlePattern.matcher(line).find()) { - String[] split = line.split("\\s+"); - - String idle = split[indexMap.get("%util")]; - - String diskName = split[indexMap.get("DEV")]; - Map> diskMap = averageMap.computeIfAbsent(diskName, e -> new HashMap<>()); - diskMap.computeIfAbsent("idle", e -> new ArrayList<>()).add(idle); - diskMap.computeIfAbsent("evidence", e -> new ArrayList<>()).add(line); - } else if (flag && StringUtils.contains(line, "DEV") || StringUtils.contains(line, "dev")) { - indexMap = getIndexMap(line); - flag = false; - } - } - } - return averageMap; - } - - private Map getIndexMap(String line) { - Map indexMap = new HashMap<>(); - String[] split = line.split("\\s+"); - for (int i =0; i < split.length; i++) { - if (split[i].equals("DEV")) { - indexMap.put("DEV", i); - } else if (split[i].equals("%util")) { - indexMap.put("%util", i); - } - } - return indexMap; - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseMemComFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseMemComFiles.java deleted file mode 100644 index aa500d33da3d9dd9b392ce665900b5e6d696d2b5..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseMemComFiles.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * ParseMemComFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseMemComFiles implements CompatibilityFilesParser { - private static final String MEMORY_CN = "内存"; - private static final String MEMORY_EN = "memory"; - private static final Pattern PATTERN = Pattern.compile("\\d+\\.\\d+"); - - private final ParseCompatibilityFilesImpl parseCompatibilityFiles; - - /** - * ParseMemComFiles - * - * @param beginMemByteMap beginMemByteMap - * @param endMemByteMap endMemByteMap - */ - public ParseMemComFiles(Map beginMemByteMap, Map endMemByteMap) { - this.parseCompatibilityFiles = new ParseCompatibilityFilesImpl(beginMemByteMap, endMemByteMap); - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - return parseCompatibilityFiles.getResult(Constants.CompatibilityTestName.IDLE_MEM.getTestName(), MEMORY_CN, - MEMORY_EN, 3); - } - - @Override - public CompatibilityTestResult parseFiles() { - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.IDLE_MEM.getTestName()); - try { - testResult = getResult(); - } catch (BaseException ex) { - log.error("Missing idle test memory file ", ex.getMessage()); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(3), MEMORY_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(3), MEMORY_EN)); - } catch (NumberFormatException | IOException ex) { - log.error("Exception occur in parsing idle test memory file ", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(2), MEMORY_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(2), MEMORY_EN)); - } - return testResult; - } - - private static class ParseCompatibilityFilesImpl extends AbsParseCompatibilityFiles { - /** - * ParseCompatibilityFilesImpl - * - * @param beginByteMap beginByteMap - * @param endByteMap endByteMap - */ - protected ParseCompatibilityFilesImpl(Map beginByteMap, Map endByteMap) { - super(beginByteMap, endByteMap); - } - - @Override - CompatibilityTestResult readFile(byte[] beginBytes, byte[] endBytes) throws IOException { - if (beginBytes.length == 0 || endBytes.length == 0) { - throw new BaseException("解析空载测试内存文件缺失"); - } - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.IDLE_MEM.getTestName()); - List beginAverageList = getAverage(beginBytes); - List endAverageList = getAverage(endBytes); - double memMaxUsedIdle = 0.0d; - List evidence; - List evidenceEn; - if (beginAverageList.size() > 0 && endAverageList.size() > 0) { - memMaxUsedIdle = new BigDecimal(beginAverageList.get(0)) - .subtract(new BigDecimal(endAverageList.get(0))) - .abs().doubleValue(); - if (memMaxUsedIdle <= 1.0) { - testResult.setResult(Constants.TEST_PASSED); - evidence = Arrays.asList(beginAverageList.get(1), endAverageList.get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(1), MEMORY_CN, - memMaxUsedIdle, "小于")); - evidenceEn = Arrays.asList(beginAverageList.get(1), endAverageList.get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(1), MEMORY_EN, - memMaxUsedIdle, "less")); - } else { - testResult.setResult(Constants.TEST_FAILED); - evidence = Arrays.asList(beginAverageList.get(1), endAverageList.get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(1), MEMORY_CN, - memMaxUsedIdle, "大于")); - evidenceEn = Arrays.asList(beginAverageList.get(1), endAverageList.get(1), - String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(1), MEMORY_EN, - memMaxUsedIdle, "greater")); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(1), - MEMORY_CN, memMaxUsedIdle, "大于")); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(1), - MEMORY_EN, memMaxUsedIdle, "greater")); - } - testResult.setEvidence(evidence); - testResult.setEvidenceEn(evidenceEn); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(2), MEMORY_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(2), - MEMORY_EN)); - } - return testResult; - } - - @Override - List getAverage(byte[] fileBytes) { - try (InputStream beginInput = new ByteArrayInputStream(fileBytes); - BufferedReader br = new BufferedReader( - new InputStreamReader(beginInput, StandardCharsets.UTF_8))) { - String line; - List averageList = new ArrayList<>(); - while ((line = br.readLine()) != null) { - average(line, averageList); - } - return averageList; - } catch (IOException e) { - throw new BaseException(e.getMessage()); - } - } - - private void average(String line, List averageList) { - if (StringUtils.containsIgnoreCase(line, "平均时间") - || StringUtils.startsWithIgnoreCase(line, - "Average")) { - Matcher matcher = PATTERN.matcher(line); - if (matcher.find()) { - averageList.add(matcher.group()); - averageList.add(line.replace(System.lineSeparator(), "")); - } - } - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseMemPerFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseMemPerFiles.java deleted file mode 100644 index d377d4eb010b0532a2e501c0c1de151e8ad1f5b3..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseMemPerFiles.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * ParseMemPerFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseMemPerFiles implements CompatibilityFilesParser { - private static final String MEMORY_CN = "内存"; - private static final String MEMORY_EN = "memory"; - - private final ParsePerformanceFilesImpl parsePerformanceFiles; - - /** - * construction file - * - * @param inputByteMap map - */ - public ParseMemPerFiles(Map inputByteMap) { - this.parsePerformanceFiles = new ParsePerformanceFilesImpl(inputByteMap); - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - return parsePerformanceFiles.getResult(Constants.CompatibilityTestName.PRESSURE_MEM.getTestName(), MEMORY_CN, - MEMORY_EN, 2); - } - - @Override - public CompatibilityTestResult parseFiles() { - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.PRESSURE_MEM.getTestName()); - try { - testResult = getResult(); - } catch (BaseException ex) { - log.error("Missing pressure test memory file ", ex.getMessage()); - testResult.setResult(Constants.TEST_SKIPPED); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(3), MEMORY_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(3), MEMORY_EN)); - } catch (NumberFormatException | IOException ex) { - log.error("Exception in parsing pressure test memory file ", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(2), MEMORY_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(2), MEMORY_EN)); - } - return testResult; - } - - private static class ParsePerformanceFilesImpl extends AbsParsePerformanceFiles { - private final Pattern idlePattern = Pattern.compile("\\d{2}(:|时)\\d{2}(:|分)\\d{2}(秒|\\s(PM|AM|HKT|))\\s+"); - private final Pattern pattern = Pattern.compile("\\d+\\.\\d+"); - - /** - * inputByteMap - * - * @param inputByteMap inputByteMap - */ - public ParsePerformanceFilesImpl(Map inputByteMap) { - super(inputByteMap); - } - - @Override - CompatibilityTestResult readFile(byte[] fileBytes) throws IOException { - if (fileBytes.length == 0) { - throw new BaseException("解析压力测试硬盘文件缺失"); - } - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.PRESSURE_MEM.getTestName()); - List idleList = getIdleList(fileBytes); - if (idleList.size() > 0) { - List evidence = new ArrayList<>(idleList.subList(1, idleList.size())); - List evidenceEn = new ArrayList<>(idleList.subList(1, idleList.size())); - if (Double.parseDouble(idleList.get(0)) <= 5.0d) { - testResult.setResult(Constants.TEST_PASSED); - evidence.add(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(0), MEMORY_CN, - Double.parseDouble(idleList.get(0)), "小于")); - evidenceEn.add(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(0), MEMORY_EN, - Double.parseDouble(idleList.get(0)), "less")); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(0), MEMORY_CN, - Double.parseDouble(idleList.get(0)), "大于")); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(0), - MEMORY_EN, - Double.parseDouble(idleList.get(0)), "greater")); - evidence.add(testResult.getReason()); - evidenceEn.add(testResult.getReasonEn()); - } - testResult.setEvidence(evidence); - testResult.setEvidenceEn(evidenceEn); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(2), MEMORY_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(2), - MEMORY_EN)); - } - return testResult; - } - - @Override - List getIdleList(byte[] fileBytes) throws IOException { - Double maxIdle = 0.0d; - String maxEvidence = null; - Double minIdle = 100.0d; - String minEvidence = null; - try (InputStream beginInput = new ByteArrayInputStream(fileBytes); - BufferedReader beginReader = new BufferedReader( - new InputStreamReader(beginInput, StandardCharsets.UTF_8))) { - String line; - while ((line = beginReader.readLine()) != null) { - Matcher matcher = pattern.matcher(line); - if (!StringUtils.containsAnyIgnoreCase(line, "kbmemfree") - && idlePattern.matcher(line).find() && matcher.find()) { - Double idleDouble = Double.parseDouble(matcher.group().trim()); - maxEvidence = maxIdle <= idleDouble ? line : maxEvidence; - maxIdle = Math.max(idleDouble, maxIdle); - minIdle = Math.min(idleDouble, minIdle); - minEvidence = minIdle >= idleDouble ? line : minEvidence; - } - } - } - - List idleList = new ArrayList<>(); - if (maxEvidence != null) { - idleList.add(String.format(Locale.ROOT, "%.2f", new BigDecimal(String.valueOf(maxIdle)) - .subtract(new BigDecimal(String.valueOf(maxIdle))).abs().doubleValue())); - idleList.add(maxEvidence); - idleList.add(minEvidence); - } - return idleList; - } - - @Override - Map>> getIdleMap(byte[] fileBytes) { - return new HashMap<>(); - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseNetComFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseNetComFiles.java deleted file mode 100644 index cb09b8fbc1be03387180c3369b2da73a06775b35..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseNetComFiles.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * ParseNetComFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseNetComFiles implements CompatibilityFilesParser { - private static final String NIC_CN = "网卡"; - private static final String NIC_EN = "NIC"; - - private final ParseCompatibilityFilesImpl parseCompatibilityFiles; - - /** - * construction function - * - * @param beginNetByteMap beginNetByteMap - * @param endNetByteMap endNetByteMap - */ - public ParseNetComFiles(Map beginNetByteMap, Map endNetByteMap) { - this.parseCompatibilityFiles = new ParseCompatibilityFilesImpl(beginNetByteMap, endNetByteMap); - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - return parseCompatibilityFiles.getResult(Constants.CompatibilityTestName.IDLE_NET.getTestName(), - NIC_CN, NIC_EN, 2); - } - - @Override - public CompatibilityTestResult parseFiles() { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.IDLE_NET.getTestName()); - try { - testResult = getResult(); - } catch (BaseException ex) { - log.error("Missing idle test net file ", ex.getMessage()); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(3), NIC_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(3), NIC_EN)); - } catch (NumberFormatException | IOException ex) { - log.error("Exception in parsing idle test net file ", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(2), NIC_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(2), NIC_EN)); - } - return testResult; - } - - private static class ParseCompatibilityFilesImpl extends AbsParseCompatibilityFiles { - private final Pattern averagePattern = Pattern.compile("(平均时间|Average):\\s*(?!IFACE|lo)\\w.*"); - private final Pattern pattern = Pattern.compile("\\d+\\.\\d+"); - - /** - * ParseCompatibilityFilesImpl - * - * @param beginByteMap beginByteMap - * @param endByteMap endByteMap - */ - protected ParseCompatibilityFilesImpl(Map beginByteMap, Map endByteMap) { - super(beginByteMap, endByteMap); - } - - @Override - CompatibilityTestResult readFile(byte[] beginBytes, byte[] endBytes) throws IOException { - if (beginBytes.length == 0 || endBytes.length == 0) { - throw new BaseException("解析空载测试网卡文件缺失"); - } - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.IDLE_NET.getTestName()); - Map> beginAverageMap = getAverageMap(beginBytes); - Map> endAverageMap = getAverageMap(endBytes); - - if (beginAverageMap.size() > 0 && endAverageMap.size() > 0) { - createTestResult(testResult, beginAverageMap, endAverageMap); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_LIST.get(2), NIC_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_EN_LIST.get(2), NIC_EN)); - } - return testResult; - } - - private void createTestResult(CompatibilityTestResult testResult, Map> beginAverageMap, - Map> endAverageMap) { - Double rxkbIdle; - Double txkbIdle; - Double maxRxkbIdle = 0.0d; - Double maxTxkbIdle = 0.0d; - String rxkbNetName = null; - String txkbNetName = null; - BigDecimal num = new BigDecimal(8).divide(new BigDecimal(1000 * 1024)); - for (Map.Entry> entry : beginAverageMap.entrySet()) { - if (!endAverageMap.containsKey(entry.getKey())) { - continue; - } - rxkbIdle = new BigDecimal(beginAverageMap.get(entry.getKey()).get(0)) - .subtract(new BigDecimal(endAverageMap.get(entry.getKey()).get(0))) - .abs().multiply(num).doubleValue(); - txkbIdle = new BigDecimal(beginAverageMap.get(entry.getKey()).get(1)) - .subtract(new BigDecimal(endAverageMap.get(entry.getKey()).get(1))) - .abs().multiply(num).doubleValue(); - if (rxkbIdle >= maxRxkbIdle) { - maxRxkbIdle = rxkbIdle; - rxkbNetName = entry.getKey(); - } - if (txkbIdle >= maxTxkbIdle) { - maxTxkbIdle = txkbIdle; - txkbNetName = entry.getKey(); - } - } - List evidence = new ArrayList<>(); - evidence.add(beginAverageMap.get(rxkbNetName).get(2)); - evidence.add(endAverageMap.get(rxkbNetName).get(2)); - List evidenceEn = new ArrayList<>(evidence); - - int[] kbpass = {0, 0}; - checkMaxRxkbIdle(maxRxkbIdle, kbpass, evidence, evidenceEn); - evidence.add(beginAverageMap.get(txkbNetName).get(2)); - evidence.add(endAverageMap.get(txkbNetName).get(2)); - evidenceEn.add(beginAverageMap.get(txkbNetName).get(2)); - evidenceEn.add(endAverageMap.get(txkbNetName).get(2)); - checkMaxTxkbIdle(maxTxkbIdle, kbpass, evidence, evidenceEn); - - testResult.setEvidence(evidence); - testResult.setEvidenceEn(evidenceEn); - setTestResult(maxRxkbIdle, maxTxkbIdle, kbpass, testResult); - } - - private void checkMaxRxkbIdle(Double maxRxkbIdle, int[] kbpass, List evidence, - List evidenceEn) { - Double maxRxkb = new BigDecimal(maxRxkbIdle.toString()).multiply(new BigDecimal(100)).doubleValue(); - if (maxRxkbIdle <= 0.01) { - kbpass[0] = 1; - evidence.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(6), - "接收", maxRxkb, "小于")); - evidenceEn.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(6), - "received", maxRxkb, "less")); - } else { - evidence.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(6), - "接收", maxRxkb, "大于")); - evidenceEn.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(6), - "received", maxRxkb, "greater")); - } - } - - private void checkMaxTxkbIdle(Double maxTxkbIdle, int[] kbpass, List evidence, - List evidenceEn) { - Double maxTxkb = new BigDecimal(maxTxkbIdle.toString()).multiply(new BigDecimal(100)).doubleValue(); - if (maxTxkbIdle < 0.01) { - kbpass[1] = 1; - evidence.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(6), "发送", - maxTxkb, "小于")); - evidenceEn.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(6), "sent", - maxTxkb, "less")); - } else { - evidence.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_LIST.get(6), "发送", - maxTxkb, "大于")); - evidenceEn.add(String.format(Locale.ROOT, Constants.COMPATIBILITY_DESC_EN_LIST.get(6), "sent", - maxTxkb, "greater")); - } - } - - private void setTestResult(Double maxRxkbIdle, Double maxTxkbIdle, int[] kbpass, - CompatibilityTestResult testResult) { - if (kbpass[0] == 1 && kbpass[1] == 1) { - testResult.setResult(Constants.TEST_PASSED); - } else { - double maxRxkb = new BigDecimal(maxRxkbIdle.toString()).multiply(new BigDecimal(100)).doubleValue(); - double maxTxkb = new BigDecimal(maxTxkbIdle.toString()).multiply(new BigDecimal(100)).doubleValue(); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_LIST.get(7), "发送", - maxTxkb) + String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_LIST.get(7), "接收", - maxRxkb)); - testResult.setReasonEn(String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_EN_LIST.get(7), - "sent", maxTxkb) + String.format(Locale.ROOT, - Constants.COMPATIBILITY_DESC_LIST.get(7), "received", - maxRxkb)); - } - } - - @Override - List getAverage(byte[] fileBytes) { - return new ArrayList<>(); - } - - /** - * get Average Map - * - * @param fileBytes fileBytes - * @return Map - * @throws IOException IOException - */ - public Map> getAverageMap(byte[] fileBytes) throws IOException { - Map> averageMap = new HashMap<>(); - try (InputStream beginInput = new ByteArrayInputStream(fileBytes); - BufferedReader beginReader = new BufferedReader( - new InputStreamReader(beginInput, StandardCharsets.UTF_8))) { - String line; - - while ((line = beginReader.readLine()) != null) { - if (!averagePattern.matcher(line).find()) { - continue; - } - Matcher matcher = pattern.matcher(line); - String netName = line.substring(13).trim().split(" ")[0]; - List matchList = new ArrayList<>(); - while (matcher.find()) { - matchList.add(matcher.group().trim()); - } - averageMap.put(netName, Arrays.asList(matchList.get(2), matchList.get(3), line)); - } - } - return averageMap; - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseNetPerFiles.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseNetPerFiles.java deleted file mode 100644 index e291033f87d514848c96a1fd8af0075c0f03da92..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseNetPerFiles.java +++ /dev/null @@ -1,265 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * ParseNetPerFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseNetPerFiles implements CompatibilityFilesParser { - private static final String NIC_CN = "网卡"; - private static final String NIC_EN = "NIC"; - - private final ParsePerformanceFilesImpl parsePerformanceFiles; - - /** - * construction function - * - * @param inputByteMap map - */ - public ParseNetPerFiles(Map inputByteMap) { - this.parsePerformanceFiles = new ParsePerformanceFilesImpl(inputByteMap); - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - return parsePerformanceFiles.getResult(Constants.CompatibilityTestName.PRESSURE_NET.getTestName(), NIC_CN, - NIC_EN, 2); - } - - @Override - public CompatibilityTestResult parseFiles() throws IOException { - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.PRESSURE_NET.getTestName()); - try { - testResult = getResult(); - } catch (BaseException ex) { - log.error("Missing pressure test net file ", ex.getMessage()); - testResult.setResult(Constants.TEST_SKIPPED); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(3), NIC_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(3), NIC_EN)); - } catch (NumberFormatException | IOException ex) { - log.error("Exception in parsing idle test net file ", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(2), NIC_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(2), NIC_EN)); - } - return testResult; - } - - @Data - private static class XxkbRate { - double maxXxkbRate = 0.00d; - String maxXxkbDiskName = null; - String netXxkbMax = null; - String netXxkbMin = null; - double netXxkbRate = 0.0d; - } - - private static class ParsePerformanceFilesImpl extends AbsParsePerformanceFiles { - private final Pattern idlePattern = Pattern.compile("\\d{2}(:|时)\\d{2}(:|分)\\d{2}(秒|\\s(PM|AM|HKT|))\\s+"); - private final Pattern pattern = Pattern.compile("\\d+\\.\\d+"); - - public ParsePerformanceFilesImpl(Map inputByteMap) { - super(inputByteMap); - } - - @Override - CompatibilityTestResult readFile(byte[] fileBytes) throws IOException { - if (fileBytes.length == 0) { - throw new BaseException("解析压力测试硬盘文件缺失"); - } - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.PRESSURE_NET.getTestName()); - Map>> netIdleMap = getIdleMap(fileBytes); - if (netIdleMap.size() > 0) { - XxkbRate rxkbRate = new XxkbRate(); - XxkbRate txkbRate = new XxkbRate(); - for (Map.Entry>> entry : netIdleMap.entrySet()) { - if (entry.getValue().get("rxkb").size() < 0 || entry.getValue().get("txkb").size() < 0) { - continue; - } - - List rxkbDoubleList = entry.getValue().get("rxkb").stream() - .map(Double::parseDouble).collect(Collectors.toList()); - calculateXxkbRate(rxkbDoubleList, entry.getKey(), rxkbRate); - - List txkbDoubleList = entry.getValue().get("txkb") - .stream().map(Double::parseDouble).collect(Collectors.toList()); - - calculateXxkbRate(txkbDoubleList, entry.getKey(), txkbRate); - } - testResult = getTestCaseResult(rxkbRate, txkbRate, netIdleMap); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(2), NIC_CN)); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(2), NIC_EN)); - } - return testResult; - } - - private void calculateXxkbRate(List rxkbDoubleList, String key, XxkbRate rate) { - Double maxRxkb = Collections.max(rxkbDoubleList); - rxkbDoubleList.remove(maxRxkb); - Double minRxkb = Collections.min(rxkbDoubleList); - rxkbDoubleList.remove(minRxkb); - double rxkbAvg = rxkbDoubleList.stream().mapToDouble(Double::valueOf).average().orElse(0.00); - if (Math.round(rxkbAvg) > 0) { - BigDecimal num = new BigDecimal(8).divide(new BigDecimal(1000 * 1024)); - rate.setNetXxkbRate(BigDecimal.valueOf(Collections.max(rxkbDoubleList)) - .subtract(new BigDecimal(String.valueOf(rxkbAvg))).abs().multiply(num).doubleValue()); - } else { - rate.setNetXxkbRate(0.0); - } - if (rate.getMaxXxkbRate() <= rate.getNetXxkbRate()) { - rate.setMaxXxkbDiskName(key); - rate.setMaxXxkbRate(rate.getNetXxkbRate()); - rate.setNetXxkbMax(String.format(Locale.ROOT, "%.2f", Collections.max(rxkbDoubleList))); - rate.setNetXxkbMin(String.format(Locale.ROOT, "%.2f", Collections.min(rxkbDoubleList))); - } - } - - private CompatibilityTestResult getTestCaseResult(XxkbRate rxkb, XxkbRate txkb, - Map>> netIdleMap) { - int[] kbPass = {0, 0}; - if (rxkb.getMaxXxkbRate() <= 0.05) { - kbPass[0] = 1; - } - if (txkb.getMaxXxkbRate() <= 0.05) { - kbPass[1] = 1; - } - - List evidence = evidenceInit(rxkb, txkb, netIdleMap); - List evidenceEn = new ArrayList<>(evidence); - Double maxRxkb = new BigDecimal(String.valueOf(rxkb.getMaxXxkbRate())) - .multiply(new BigDecimal(100)).doubleValue(); - Double maxTxkb = new BigDecimal(String.valueOf(txkb.getMaxXxkbRate())) - .multiply(new BigDecimal(100)).doubleValue(); - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.PRESSURE_NET.getTestName()); - if (kbPass[0] == 1 && kbPass[1] == 1) { - testResult.setResult(Constants.TEST_PASSED); - evidence.add(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_LIST.get(4), "接收", - maxRxkb) + String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(5), "发送", - maxTxkb, "小于")); - evidenceEn.add(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(4), "received", - maxRxkb) + String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(5), "sent", - maxTxkb, "less")); - } else { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(4), "接收", - maxRxkb) + String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_LIST.get(5), "发送", - maxTxkb, "大于")); - testResult.setReasonEn(String.format(Locale.ROOT, Constants.PERFORMANCE_DESC_EN_LIST.get(4), - "received", maxRxkb) + String.format(Locale.ROOT, - Constants.PERFORMANCE_DESC_EN_LIST.get(5), "sent", - maxTxkb, "greater")); - evidence.add(testResult.getResult()); - evidenceEn.add(testResult.getReasonEn()); - } - testResult.setEvidence(evidence); - testResult.setEvidenceEn(evidenceEn); - return testResult; - } - - private List evidenceInit(XxkbRate rxkb, XxkbRate txkb, - Map>> netIdleMap) { - List rxkbEvidenceList = netIdleMap.get(rxkb.getMaxXxkbDiskName()).get("evidence"); - List txkbEvidenceList = netIdleMap.get(txkb.getMaxXxkbDiskName()).get("evidence"); - - List evidence = new ArrayList<>(); - String finalNetRxkbMax = rxkb.getNetXxkbMax(); - evidence.add(rxkbEvidenceList.stream().filter(s -> StringUtils.containsIgnoreCase(s, finalNetRxkbMax)) - .limit(1).collect(Collectors.joining())); - String finalNetRxkbMin = rxkb.getNetXxkbMin(); - evidence.add(rxkbEvidenceList.stream().filter(s -> StringUtils.containsIgnoreCase(s, finalNetRxkbMin)) - .limit(1).collect(Collectors.joining())); - String finalNetTxkbMax = txkb.getNetXxkbMax(); - evidence.add(txkbEvidenceList.stream().filter(s -> StringUtils.containsIgnoreCase(s, finalNetTxkbMax)) - .limit(1).collect(Collectors.joining())); - String finalNetTxkbMin = txkb.getNetXxkbMin(); - evidence.add(txkbEvidenceList.stream().filter(s -> StringUtils.containsIgnoreCase(s, finalNetTxkbMin)) - .limit(1).collect(Collectors.joining())); - return evidence; - } - - @Override - List getIdleList(byte[] fileBytes) throws IOException { - return new ArrayList<>(); - } - - @Override - Map>> getIdleMap(byte[] fileBytes) throws IOException { - Map>> averageMap = new HashMap<>(); - try (InputStream beginInput = new ByteArrayInputStream(fileBytes); - BufferedReader beginReader = new BufferedReader( - new InputStreamReader(beginInput, StandardCharsets.UTF_8))) { - String line; - - while ((line = beginReader.readLine()) != null) { - if(StringUtils.containsIgnoreCase(line, "IFACE") || StringUtils.containsIgnoreCase(line, "lo")){ - continue; - } - if (!idlePattern.matcher(line).find()){ - continue; - } - Matcher matcher = pattern.matcher(line); - String netName = line.substring(13).trim().split(" ")[0]; - List matchList = new ArrayList<>(); - while (matcher.find()) { - matchList.add(matcher.group().trim()); - } - Map> netMap = averageMap.getOrDefault(netName, new HashMap<>()); - List rxkbList = netMap.getOrDefault("rxkb", new ArrayList<>()); - rxkbList.add(matchList.get(2)); - List txkbList = netMap.getOrDefault("txkb", new ArrayList<>()); - txkbList.add(matchList.get(3)); - List evidenceList = netMap.getOrDefault("evidence", new ArrayList<>()); - evidenceList.add(line); - netMap.put("rxkb", rxkbList); - netMap.put("txkb", txkbList); - netMap.put("evidence", evidenceList); - averageMap.put(netName, netMap); - } - } - return averageMap; - } - } -} \ No newline at end of file diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseSoftwareLog.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseSoftwareLog.java deleted file mode 100644 index 75c35cb060a799f79d232613d0b98b986e275012..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/entity/compatibilityfileanalysis/impl/ParseSoftwareLog.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityFilesParser; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * ParseNetPerFiles - * - * @author kongcaizhi - * @since 2021-12-15 - */ -@Slf4j -public class ParseSoftwareLog implements CompatibilityFilesParser { - private final byte[] configBytes; - private final byte[] productNameBytes; - - /** - * construction function - * - * @param configBytes config bytes - * @param productNameBytes product name bytes - */ - public ParseSoftwareLog(byte[] configBytes, byte[] productNameBytes) { - this.configBytes = configBytes; - this.productNameBytes = productNameBytes; - } - - private Map> parseConfig() { - Map> configMap = new HashMap<>(); - if (configBytes.length == 0) { - return configMap; - } - - String line; - Pattern appPattern = Pattern.compile(Constants.APPLICATION_NAMES); - Pattern startPattern = Pattern.compile(Constants.START_APP_COMMANDS); - Pattern stopPattern = Pattern.compile(Constants.STOP_APP_COMMANDS); - try (InputStream input = new ByteArrayInputStream(configBytes); - BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) { - while ((line = reader.readLine()) != null) { - Matcher appMatcher = appPattern.matcher(line); - if (appMatcher.find()) { - List applicationNamesList = Arrays.asList(appMatcher.group(1).split(",")); - configMap.put("applicationNames", applicationNamesList); - } - - Matcher startMatcher = startPattern.matcher(line); - if (startMatcher.find()) { - List startAppCommandsList = Arrays.asList(startMatcher.group(1).split(",")); - configMap.put("startAppCommands", startAppCommandsList); - } - - Matcher stopMatcher = stopPattern.matcher(line); - if (stopMatcher.find()) { - List stopAppCommandsList = Arrays.asList(stopMatcher.group(1).split(",")); - configMap.put("stopAppCommands", stopAppCommandsList); - } - } - } catch (IllegalStateException | IOException ex) { - log.error("Error reading configuration file for compatibility testing tool", ex); - } - return configMap; - } - - @Override - public CompatibilityTestResult getResult() throws IOException { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.SOFTWARE_NAME.getTestName()); - if (this.productNameBytes.length == 0) { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SOFTWARE_COMPARE_DESC_LIST.get(3)); - return testResult; - } - Map> configMap = parseConfig(); - if (configMap.size() == 0 || configMap.get("applicationNames").size() == 0) { - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SOFTWARE_COMPARE_DESC_LIST.get(2)); - } else { - testResult.setApplicationNames(configMap.get("applicationNames")); - testResult.setStartAppCommands(configMap.get("startAppCommands")); - testResult.setStopAppCommands(configMap.get("stopAppCommands")); - } - - getApplicationNames(configMap, testResult, this.productNameBytes); - return testResult; - } - - private void getApplicationNames(Map> configMap, CompatibilityTestResult testResult, - byte[] productNameBytes) { - try (InputStream inputStream = new ByteArrayInputStream(productNameBytes); - BufferedReader reader = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { - String line; - Map appCountMap = new HashMap<>(); - while ((line = reader.readLine()) != null) { - getApplicationName(line, configMap, testResult, appCountMap); - } - for (Map.Entry entry : appCountMap.entrySet()) { - if (appCountMap.get(entry.getKey()) == 0) { - testResult.setResult(Constants.TEST_FAILED); - List evidenceList = testResult.getEvidence(); - evidenceList.add(String.format(Locale.ROOT, - Constants.SOFTWARE_COMPARE_DESC_LIST.get(0), entry.getKey())); - testResult.setEvidence(evidenceList); - evidenceList = testResult.getEvidenceEn(); - evidenceList.add(String.format(Locale.ROOT, - Constants.SOFTWARE_COMPARE_DESC_EN_LIST.get(0), entry.getKey())); - testResult.setEvidenceEn(evidenceList); - } - } - if (Constants.TEST_FAILED.equals(testResult.getResult())) { - testResult.setReason(String.format(Locale.ROOT, - Constants.SOFTWARE_COMPARE_DESC_LIST.get(1), configMap.get("applicationNames").toString())); - testResult.setReasonEn(String.format(Locale.ROOT, - Constants.SOFTWARE_COMPARE_DESC_EN_LIST.get(1), configMap.get("applicationNames").toString())); - } else { - testResult.setResult(Constants.TEST_PASSED); - } - } catch (IOException e) { - throw new BaseException(e.getMessage()); - } - } - - private void getApplicationName(String line, Map> configMap, - CompatibilityTestResult testResult, Map appCountMap) { - int userLength = line.split(" ")[0].length(); - String lineSplit = line.substring(userLength); - - for (String app : configMap.get("applicationNames")) { - if (StringUtils.startsWithIgnoreCase(lineSplit, app)) { - appCountMap.put(app, appCountMap.getOrDefault(app, 0) + 1); - List evidenceList = testResult.getEvidence() == null ? new ArrayList<>() - : testResult.getEvidence(); - evidenceList.add(line.replace(System.lineSeparator(), "")); - testResult.setEvidence(evidenceList); - testResult.setEvidenceEn(evidenceList); - } - } - } - - @Override - public CompatibilityTestResult parseFiles() { - CompatibilityTestResult testResult = new CompatibilityTestResult( - Constants.CompatibilityTestName.SOFTWARE_NAME.getTestName()); - try { - testResult = getResult(); - } catch (IllegalStateException | IOException ex) { - log.error("Error parsing configuration file and software process stack information of compatibility " - + "testing tool", ex); - testResult.setResult(Constants.TEST_FAILED); - testResult.setReason(Constants.SOFTWARE_COMPARE_DESC_LIST.get(4)); - testResult.setReasonEn(Constants.SOFTWARE_COMPARE_DESC_EN_LIST.get(4)); - } - return testResult; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/DevkitService.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/DevkitService.java deleted file mode 100644 index 51ef42d0d1771ddc5ac29c553f6182f624687738..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/DevkitService.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.service; - -import com.huawei.ic.openlab.cloudtest.entity.SystemParams; -import com.huawei.ic.openlab.cloudtest.util.ApiCenterUtil; -import com.huawei.ic.openlab.cloudtest.util.DgCodeServerClient; -import com.huawei.ic.openlab.cloudtest.util.TeeCodeServerClient; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.stereotype.Service; -import org.springframework.web.context.request.RequestAttributes; -import org.springframework.web.context.request.RequestContextHolder; -import org.springframework.web.context.request.ServletRequestAttributes; - -import java.util.concurrent.TimeUnit; - -import javax.servlet.http.HttpServletRequest; - -/** - * devkit环境申请service - * - * @author kongcaizhi - * @since 2022-10-19 - */ -@Slf4j -public class DevkitService { - private final TeeCodeServerClient client; - private final DgCodeServerClient dgCodeServerClient; - private final SystemParams systemParams; - - @Autowired - private RedisTemplate redisTemplate; - - /** - * construction function - * - * @param client client - * @param dgCodeServerClient dgCodeServer client - * @param systemParams system parameter - */ - public DevkitService(TeeCodeServerClient client, DgCodeServerClient dgCodeServerClient, - SystemParams systemParams) { - this.client = client; - this.dgCodeServerClient = dgCodeServerClient; - this.systemParams = systemParams; - } - - /** - * 往redis 增加 device-id - * - * @param server 服务器信息 - */ - public void addTaskDeviceId(DevkitServer server) { - client.addTaskDeviceId(server.getTaskDeviceId(), server.getInternalIp()); - dgCodeServerClient.addTaskDeviceId(server.getTaskDeviceId(), server.getInternalIp()); - if (Boolean.TRUE.equals(redisTemplate.hasKey(server.getTaskDeviceId()))) { - redisTemplate.opsForValue().setIfAbsent(server.getTaskDeviceId(), server.getInternalIp()); - } else { - redisTemplate.opsForValue().append(server.getTaskDeviceId(), server.getInternalIp()); - } - } - - /** - * 往redis 删除device-id - * - * @param taskDeviceId device-id - * @return boolean - */ - public boolean deleteTaskDeviceId(String taskDeviceId) { - client.deleteTaskDeviceId(taskDeviceId); - dgCodeServerClient.deleteTaskDeviceId(taskDeviceId); - return Boolean.TRUE.equals(redisTemplate.delete(taskDeviceId)); - } - - /** - * 向API网关验证token是否有效 - * - * @return validate - */ - public boolean validateToken() { - RequestAttributes requestAttributes = RequestContextHolder.getRequestAttributes(); - HttpServletRequest request = requestAttributes instanceof ServletRequestAttributes - ? ((ServletRequestAttributes) requestAttributes).getRequest() : null; - - if (request == null) { - return false; - } - - String token = request.getHeader("KP-TOKEN"); - String taskDeviceId = request.getHeader("TASK-DEVICE-ID"); - - if (StringUtils.isEmpty(token)) { - log.error("token is empty"); - return false; - } - - if (StringUtils.isEmpty(taskDeviceId)) { - log.error("taskDeviceId is empty"); - return false; - } - return validateToken(token, taskDeviceId); - } - - /** - * 检查redis是否有token, 如果有,返回为真,如果没有请求apigw判断.如果apigw返回为真,则保存在redis中,设置超时时间为1分钟. - * - * @param token token - * @param taskDeviceId taskDeviceId - * @return true|false - */ - private boolean validateToken(String token, String taskDeviceId) { - String key = token + "-" + taskDeviceId; - - if (Boolean.TRUE.equals(redisTemplate.hasKey(key))) { - return true; - } else { - boolean isValidated = ApiCenterUtil.validateToken(systemParams, token, taskDeviceId); - if (isValidated) { - redisTemplate.opsForValue().set(key, "token", 5, TimeUnit.MINUTES); - log.info("Add token to redis return{}", - redisTemplate.expire(key, 5, TimeUnit.MINUTES)); - } - return isValidated; - } - } - - /** - * DevkitServer - * - * @author kongcaizhi - * @since 2022-10-19 - */ - @Data - public static class DevkitServer { - private String taskDeviceId; - private String agentId; - private String internalIp; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/LabEnvService.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/LabEnvService.java deleted file mode 100644 index 1eba0aa4f009f59a2c76e1d66a4d857fd6822af1..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/LabEnvService.java +++ /dev/null @@ -1,642 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.service; - -import com.alibaba.fastjson.JSONArray; -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; -import com.huawei.ic.openlab.cloudtest.common.exception.SshErrorException; -import com.huawei.ic.openlab.cloudtest.entity.CloudLabTestTask; -import com.huawei.ic.openlab.cloudtest.entity.LabTestReq; -import com.huawei.ic.openlab.cloudtest.entity.MqsMessage; -import com.huawei.ic.openlab.cloudtest.entity.ScriptResultConfig; -import com.huawei.ic.openlab.cloudtest.entity.SystemParams; -import com.huawei.ic.openlab.cloudtest.entity.TestCaseResult; -import com.huawei.ic.openlab.cloudtest.entity.TestCaseResultCount; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.util.Constants; -import com.huawei.ic.openlab.cloudtest.util.FileUtil; -import com.huawei.ic.openlab.cloudtest.util.NormalResp; -import com.huawei.ic.openlab.cloudtest.util.PerformanceApiClient; -import com.huawei.ic.openlab.cloudtest.util.RandomUtil; -import com.huawei.ic.openlab.cloudtest.util.ToolUtil; -import com.huawei.ic.openlab.cloudtest.util.fastdfs.FastDfsClient; -import com.huawei.ic.openlab.cloudtest.util.sshclient.SFTPUtil; -import com.huawei.ic.openlab.cloudtest.util.sshclient.SSHUtil; - -import com.alibaba.fastjson.JSONObject; -import com.jcraft.jsch.JSchException; -import com.jcraft.jsch.SftpException; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.DigestUtils; -import org.springframework.web.multipart.MultipartFile; - -import java.io.*; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; - -/** - * LabEnvService - * - * @author kongcaizhi - * @since 2021-10-19 - */ -@Service -@Slf4j -public class LabEnvService { - @Autowired - private SystemParams systemParams; - @Autowired - private TarFileService fileService; - @Autowired - private PerformanceApiClient performanceApiClient; - @Autowired - private FastDfsClient fastDfsClient; - - /** - * getLabConnectTest - * - * @param ip ip - * @param port port - * @param userName userName - * @param passWord password - * @return boolean - */ - public boolean getLabConnectTest(String ip, int port, String userName, String passWord) { - return SSHUtil.sshConnectTest(ip, port, userName, passWord); - } - - /** - * validatedZipFileSize - * - * @param fileInputStream fileInputStream - * @param language language - * @throws IOException exception - */ - public void validatedZipFileSize(InputStream fileInputStream, String language) throws IOException { - fileService.validatedZipFileSize(fileInputStream, language); - } - - /** - * validatedTarZipFile - * - * @param fileInputStream fileInputStream - * @throws IOException exception - */ - public void validatedTarZipFile(InputStream fileInputStream) throws IOException { - fileService.validatedTarGzFileSize(fileInputStream); - } - - - /** - * testPreparation - * - * @param testReq testReq - * @throws SshErrorException SshErrorException - * @throws FileNotFoundException FileNotFoundException - */ - public void testPreparation(LabTestReq testReq) throws SshErrorException, FileNotFoundException { - List commandList = new ArrayList<>(); - commandList.add("mkdir -p /home/compatibility_testing; mkdir -p /home/function_testing"); - String wgetCom = "cd /home/compatibility_testing; wget http://%s:8080/download/compatibility_testing.tar.gz"; - commandList.add(String.format(Locale.ROOT, wgetCom, systemParams.getDeployIP())); - commandList.add("cd /home/compatibility_testing; tar -xvzf compatibility_testing.tar.gz"); - String wgetFun = "cd /home/function_testing; wget http://%s:8080/download/shunit2-master.zip"; - commandList.add(String.format(Locale.ROOT, wgetFun, systemParams.getDeployIP())); - commandList.add("cd /home/function_testing; unzip -o -q shunit2-master.zip"); - SSHUtil.sshExecCmd(testReq.getServerIp(), testReq.getServerPort(), testReq.getServerUser(), - testReq.getServerPassword(), commandList); - } - - /** - * 修改配置文件,并上传到实验室环境 - * - * @param testReq testReq - * @throws IOException IOException - * @throws JSchException JSchException - * @throws SftpException SftpException - */ - private void setConfigFile(LabTestReq testReq) throws IOException, JSchException, SftpException { - String configContent = readConfigFile(testReq); - String configFileName = writeConfigFile(configContent); - ToolUtil.validPath(configFileName); - File configFile = new File(configFileName); - try (InputStream in = Files.newInputStream(configFile.toPath())) { - String targetPath = String.format(Locale.ROOT, "/home/compatibility_testing/compatibility_testing/%s" - + "/compatibility_testing.conf", "CN".equals(testReq.getTaskLanguage()) ? "Chinese" : "English"); - SFTPUtil.uploadFile(testReq, targetPath, in); - } - log.info("Delete configure file {},result {}", configFileName, configFile.delete()); - } - - private String readConfigFile(LabTestReq testReq) { - StringBuilder res = new StringBuilder(16); - try (BufferedReader reader = new BufferedReader( - new FileReader(systemParams.getScriptConfig(testReq.getTaskLanguage())))) { - String line; - while ((line = reader.readLine()) != null) { - if (StringUtils.startsWithIgnoreCase(line, "application_names=")) { - line = line + testReq.getApplicationNames(); - } - if (StringUtils.startsWithIgnoreCase(line, "start_app_commands=")) { - line = line + testReq.getStartAppCommands(); - } - if (StringUtils.startsWithIgnoreCase(line, "stop_app_commands=")) { - line = line + testReq.getStopAppCommands(); - } - if (StringUtils.startsWithIgnoreCase(line, "application_install_dir=")) { - line = line + testReq.getDeployDir(); - } - res.append(line).append(System.lineSeparator()); - } - } catch (FileNotFoundException ex) { - log.error(ex.getLocalizedMessage()); - throw new BaseException("配置文件不存在"); - } catch (IOException e) { - log.error(e.getLocalizedMessage(), e); - } - return res.toString(); - } - - private String writeConfigFile(String content) { - SecureRandom random = RandomUtil.getRandom(); - int ends = random.nextInt(99); - String fileName = ToolUtil.getTimeStr() + String.format(Locale.ROOT, "%02d", ends); - fileName = systemParams.getTempDir() + File.separator + fileName; - try (BufferedWriter writer = new BufferedWriter(new FileWriter(fileName))) { - writer.write(content); - writer.flush(); - } catch (IOException e) { - log.error(e.getLocalizedMessage(), e); - } - return fileName; - } - - private void setStep(String step, String resultCode, String timeString, CloudLabTestTask.StepStatus stepStatus) { - switch (step) { - case "01": - stepStatus.setStep1(resultCode, timeString); - break; - case "02": - stepStatus.setStep2(resultCode, timeString); - break; - case "03": - stepStatus.setStep3(resultCode, timeString); - break; - case "04": - stepStatus.setStep4(resultCode, timeString); - break; - case "05": - stepStatus.setStep5(resultCode, timeString); - break; - case "06": - stepStatus.setStep6(resultCode, timeString); - break; - case "07": - stepStatus.setStep7(resultCode, timeString); - break; - case "08": - stepStatus.setStep8(resultCode, timeString); - break; - case "09": - stepStatus.setStep9(resultCode, timeString); - break; - default: - stepStatus.setStep10(resultCode, timeString); - break; - } - } - - private void setTestTime(String step, String timeString, CloudLabTestTask.TestBeginTime testBeginTime, - boolean isFunctionTest) { - if ("05".equals(step)) { - testBeginTime.setCompatibilityTime(timeString); - testBeginTime.setSecurityTime(timeString); - testBeginTime.setReliabilityTime(timeString); - testBeginTime.setFunctionTime(timeString); - } else if ("06".equals(step)) { - testBeginTime.setCompatibilityTime(timeString); - testBeginTime.setReliabilityTime(timeString); - } else if ("08".equals(step)) { - testBeginTime.setCompatibilityTime(timeString); - } else if ("09".equals(step) && isFunctionTest) { - testBeginTime.setFunctionTime(timeString); - } else { - log.info("not contrast"); - } - } - - /** - * 组装测试进度推送MQS 消息 - * - * @param task task - * @param resultCode resultCode - * @param timeString timeString - * @return MqsMessage object - */ - private MqsMessage getProgressMessage(CloudLabTestTask task, String resultCode, String timeString) { - MqsMessage message = new MqsMessage(); - message.setProjectId(task.getProjectId()); - message.setUserId(task.getUserId()); - message.setServerIp(task.getServerIp()); - message.setStatusTime(timeString); - message.setStatus(Constants.MQS_STATUS_ONGOING); - message.setStatusDesc(""); - - MqsMessage.MessageDetail detail = setDetail(task, resultCode); - message.setDetail(detail); - return message; - } - - private MqsMessage.MessageDetail setDetail(CloudLabTestTask task, String resultCode) { - MqsMessage.MessageDetail detail = new MqsMessage.MessageDetail(); - switch (Objects.requireNonNull(Constants.CompatibilityTestStep.fromStepIndex(resultCode))) { - case DEPENDENCY_INSTALL_SUCCESS: - detail.setDependencyInstallSuccess(); - break; - case APP_START_SUCCESS: - detail.setAppStartSuccess(); - break; - case APP_STOP_SUCCESS: - detail.setAppStopSuccess(task.isCompatibilityTest(), - task.isReliabilityTest(), task.isSecurityTest()); - break; - case COMPATIBILITY_TEST_SUCCESS: - case COM_START_SUCCESS: - detail.setComStartSuccess(task.isReliabilityTest(), task.isSecurityTest()); - if ((task.isCompatibilityTest() || task.isReliabilityTest()) - && !task.isSecurityTest() && task.getPerformanceFile() != null) { - setPerformanceTest(task, true); - } - break; - case PORT_TEST_SUCCESS: - detail.setPortTestSuccess(); - break; - case VIRUS_SCAN_SUCCESS: - detail.setVirusScanSuccess(task.isCompatibilityTest(), task.isReliabilityTest()); - if ((task.isCompatibilityTest() || task.isReliabilityTest()) && task.getPerformanceFile() != null) { - setPerformanceTest(task, true); - } - break; - case VULNERABLE_SCAN_SUCCESS: - detail.setVulnerableScanSuccess(task.isCompatibilityTest(), task.isReliabilityTest()); - break; - case EXCEPTION_TEST_FINISH: - detail.setExceptionTestFinish(task.isCompatibilityTest()); - break; - case COMPATIBILITY_TEST_FINISH: - detail.setStep(Constants.COMPATIBILITY_TEST_DESC); - detail.setStepStatus(Constants.TEST_FINISHED_CN); - break; - case FUN_TEST_FILE_EXIST_ERROR: - case FUN_TEST_FILE_DIR_ERROR: - case FUN_TEST_PYTEST_ERROR: - detail.setStep(Constants.FUNCTION_TEST_DESC); - detail.setStepStatus(Constants.TEST_FAILURE_CN); - break; - default: - break; - } - return detail; - } - - private void sendExceptionMessage(CloudLabTestTask task, String timeString, MqsMessage.MessageDetail detail) { - MqsMessage message = new MqsMessage(); - message.setProjectId(task.getProjectId()); - message.setUserId(task.getUserId()); - message.setServerIp(task.getServerIp()); - message.setStatusTime(timeString); - message.setStatus(Constants.MQS_STATUS_EXCEPTION); - message.setStatusDesc(detail.getExceptionType()); - message.setDetail(detail); - } - - private void ransfer2json(List input_info, String path) { - String jsonArray = JSONArray.toJSONString(input_info); - byte[] temp_result = jsonArray.getBytes(StandardCharsets.UTF_8); - try(FileOutputStream fos=new FileOutputStream(new File(path)); - ByteArrayInputStream bais = new ByteArrayInputStream(temp_result)){ - IOUtils.copy(bais, fos); - } catch (FileNotFoundException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public void resultJson(String multipartFile, String savePath) { - List testCaseResultList = null; - List results = fileService.compatibilityResultAnalysis(multipartFile); - if (results.size() != 0){ - testCaseResultList = analyseCompatibilityTestResult(results, "CN"); - } else { - throw new BaseException("解析兼容性测试工具输出日志异常"); - } - ransfer2json(testCaseResultList, savePath); - } - - private String resultFileAnalysis(CloudLabTestTask task, MultipartFile multipartFile, String resultCode, - String fileStatus, String timeString) { - List temp = null; - CloudLabTestTask.StepStatus stepStatus = task.getStepStatus(); - if (stepStatus == null) { - stepStatus = new CloudLabTestTask.StepStatus(); - } - stepStatus.setStep10Code(resultCode); - stepStatus.setStep10Time(timeString); - stepStatus.setStep10File(getFileFromRequest(multipartFile, "兼容性测试工具输出日志", fileStatus)); - if (!task.isPerformanceTest() || (task.getPerformanceFile() == null - && Constants.RELIABILITY_START_APP_FAIL_CODE.equals(stepStatus.getStep9Code()))) { - task.setProjectStatus(Constants.CLOUD_LAB_TEST_FINISH); - task.setFinishTime(timeString); - - task.setProjectStatus(Constants.CLOUD_LAB_TEST_FINISH); - } else { - stepStatus.setPerformanceTime(timeString); - } - if (multipartFile != null) { - task.setResultFile(stepStatus.getStep10File()); - List results = fileService.compatibilityResultAnalysis(multipartFile.getOriginalFilename()); - if (results.size() != 0) { - temp = analyseCompatibilityTestResult(results, "CN"); - } else { - task.setStatusDesc("解析兼容性测试工具输出日志异常"); - task.setProjectStatus(Constants.CLOUD_LAB_TEST_EXCEPTION); - } - } else { - task.setStatusDesc("兼容性测试工具输出日志," + fileStatus); - task.setProjectStatus(Constants.CLOUD_LAB_TEST_EXCEPTION); - } - - task.setStepStatus(stepStatus); - - return NormalResp.ok(); - } - - private void setPerformanceTest(CloudLabTestTask task, boolean isTimeLimited) { - try { - String fileName = task.getPerformanceFile().getFilePath(); - MultipartFile multipartFile = FileUtil.fileToMultipartFile(fileName); - String result = performanceApiClient.setPerformanceTest(createURI(task.getPerformanceService()), - task.getProjectId(), task.getServerIp(), isTimeLimited, multipartFile); - log.info("Task {} request {} return from performance test {}", task.getProjectId(), - task.getPerformanceService(), result); - } catch (URISyntaxException ex) { - log.error("Failed to request performance test {}", ex.getLocalizedMessage()); - } - } - - /** - * 往MQS发送性能测试中状态 - * - * @param task task - * @param timeString timeString - */ - private void sendPerformanceMQS(CloudLabTestTask task, String timeString) { - MqsMessage message = new MqsMessage(); - message.setProjectId(task.getProjectId()); - message.setUserId(task.getUserId()); - message.setServerIp(task.getServerIp()); - message.setStatusTime(timeString); - message.setStatus(Constants.MQS_STATUS_ONGOING); - message.setStatusDesc(""); - MqsMessage.MessageDetail detail = new MqsMessage.MessageDetail(); - detail.setStep(Constants.PERFORMANCE_TEST_DESC); - detail.setStepStatus(Constants.TEST_PROCESSING); - message.setDetail(detail); - } - - /** - * 往MQS发送分析结果 - * - * @param task task - * @param timeString time string - */ - private void sentResultMQSMessage(CloudLabTestTask task, String timeString) { - MqsMessage message = new MqsMessage(); - message.setProjectId(task.getProjectId()); - message.setUserId(task.getUserId()); - message.setServerIp(task.getServerIp()); - message.setStatusTime(timeString); - message.setStatus(Constants.MQS_STATUS_FINISH); - MqsMessage.MessageDetail detail = new MqsMessage.MessageDetail(); - detail.setResultFileName(task.getResultFile().getFileName()); - detail.setFileStatus(Constants.TEST_NORMAL_CN); - message.setDetail(detail); - MqsMessage.MessageResult result = new MqsMessage.MessageResult(); - result.setScriptResultConfig(task.getScriptResultConfig()); - result.setTestDetail(task.getTestDetail()); - result.setTestSummary(task.getTestSummary()); - message.setTestResult(result); - - } - - /** - * 上传的文件,存储到服务器,返回路径 - * - * @param multipartFile multipartFile - * @param type type - * @param fileStatus fileStatus - * @return 路径 - */ - private CloudLabTestTask.UploadFile getFileFromRequest(MultipartFile multipartFile, String type, - String fileStatus) { - CloudLabTestTask.UploadFile uploadFile = new CloudLabTestTask.UploadFile(); - uploadFile.setFileDesc(type); - uploadFile.setFileStatus(fileStatus); - if (multipartFile == null) { - return uploadFile; - } - try { - SecureRandom random = RandomUtil.getRandom(); - String fileName = random.getAlgorithm() + "-" - + DigestUtils.md5DigestAsHex(multipartFile.getInputStream()) - + multipartFile.getOriginalFilename().substring( - multipartFile.getOriginalFilename().lastIndexOf(".")); - String dirName = systemParams.getUploadFileDir() + File.separator - + ToolUtil.getDateStr() + File.separator + fileName; - ToolUtil.validPath(dirName); - File file = new File(dirName); - FileUtils.copyToFile(multipartFile.getInputStream(), file); - uploadFile.setFileId(DigestUtils.md5DigestAsHex(multipartFile.getInputStream())); - uploadFile.setFileName(fileName); - uploadFile.setFilePath(dirName); - } catch (IOException ex) { - log.error(ex.getLocalizedMessage(), ex); - uploadFile.setFileStatus(Constants.UPLOAD_FILE_STATUS_EXCEPTION); - } - return uploadFile; - } - - private List analyseCompatibilityTestResult(List results, String lan) { - ScriptResultConfig config = new ScriptResultConfig(); - TestCaseResultCount functionCount = new TestCaseResultCount(); - TestCaseResultCount compatibilityCount = new TestCaseResultCount(); - TestCaseResultCount reliabilityCount = new TestCaseResultCount(); - TestCaseResultCount securityCount = new TestCaseResultCount(); - List testCaseResults = new ArrayList<>(); - - for (CompatibilityTestResult item : results) { - ready(item, config); - if (StringUtils.isNoneEmpty(item.getId()) && item.getId().startsWith( - "Compatibility")) { - compatibilityCount.add(item.getResult()); - testCaseResults.add(getTestCaseResult(item, lan)); - } else if (StringUtils.isNoneEmpty(item.getId()) && item.getId().startsWith( - "Reliability")) { - reliabilityCount.add(item.getResult()); - testCaseResults.add(getTestCaseResult(item, lan)); - } else if (StringUtils.isNoneEmpty(item.getId()) && item.getId().startsWith( - "Security")) { - securityCount.add(item.getResult()); - testCaseResults.add(getTestCaseResult(item, lan)); - } else { - log.info("not contrast"); - } - } - CloudLabTestTask.TestCaseSummary testCaseSummary = new CloudLabTestTask.TestCaseSummary(compatibilityCount, - reliabilityCount, securityCount, functionCount); - - return testCaseResults; - } - - private void ready(CompatibilityTestResult item, ScriptResultConfig config) { - if (StringUtils.isNoneEmpty(item.getId()) && StringUtils.equals(item.getId(), - Constants.CompatibilityTestName.APPLICATION_START.getTestName())) { - config.setOsVersion(item.getOsVersion()); - } else if (StringUtils.isNoneEmpty(item.getId()) && StringUtils.equals(item.getId(), - Constants.CompatibilityTestName.SOFTWARE_NAME.getTestName())) { - config.setApplicationNames(item.getApplicationNames() != null ? String.join(",", - item.getApplicationNames()) : ""); - config.setStartAppCommands(item.getStartAppCommands() != null ? String.join(",", - item.getStartAppCommands()) : ""); - config.setStopAppCommands(item.getStopAppCommands() != null ? String.join(",", - item.getStopAppCommands()) : ""); - } else { - log.info("not contrast"); - } - } - - private TestCaseResult createTestCaseResult(CloudLabTestTask.UploadFile performanceFile, String taskLanguage) { - TestCaseResult testCaseResult = new TestCaseResult(); - testCaseResult.setId(Constants.PERFORMANCE_TEST_ID); - testCaseResult.setResult(Constants.TEST_FAILED); - if (performanceFile == null) { - testCaseResult.setReason(Constants.PERFORMANCE_FILE_MISSING_MAP.get(taskLanguage)); - } else { - testCaseResult.setReason(Constants.APP_START_UP_FAILURE_DESC_MAP.get(taskLanguage)); - } - return testCaseResult; - } - - private void setFunctionCount(CompatibilityTestResult item, TestCaseResultCount functionCount, - CloudLabTestTask.StepStatus status, CloudLabTestTask.TestBeginTime testBeginTime) { - if (Constants.TEST_PASSED.equals(item.getResult())) { - JSONObject jsonObject = JSONObject.parseObject(item.getReason()); - functionCount.setPassed(jsonObject.getIntValue(Constants.TEST_PASSED)); - functionCount.setFailed(jsonObject.getIntValue(Constants.TEST_FAILED)); - functionCount.setTotal(jsonObject.getIntValue(Constants.TEST_PASSED) - + jsonObject.getIntValue(Constants.TEST_FAILED)); - } - if (Constants.RELIABILITY_START_APP_FAIL_CODE.equals(status.getStep9Code())) { - functionCount.setPassed(0); - functionCount.setFailed(0); - functionCount.setTotal(0); - } - functionCount.setStartTime(testBeginTime.getFunctionTime()); - } - - private TestCaseResult getTestCaseResult(CompatibilityTestResult testResult, String language) { - TestCaseResult testCaseResult = new TestCaseResult(); - testCaseResult.setId(testResult.getId()); - testCaseResult.setResult(testResult.getResult()); - testCaseResult.setReason("CN".equals(language) ? testResult.getReason() : testResult.getReasonEn()); - testCaseResult.setEvidence(String.join(",", "CN".equals(language) - ? testResult.getEvidence() : testResult.getEvidenceEn())); - return testCaseResult; - } - - /** - * 上传功能测试文件到远程实验室 - * - * @param task task - * @param uploadFile upload file - */ - private void setRemoteFunctionFile(CloudLabTestTask task, CloudLabTestTask.UploadFile uploadFile) { - ToolUtil.validPath(uploadFile.getFilePath()); - try (InputStream in = new FileInputStream(uploadFile.getFilePath())) { - SFTPUtil.uploadFile(task, "/home/function_testing/function_testing.zip", in); - } catch (JSchException | SftpException | IOException ex) { - log.error("Failed to upload task {} function file {} to remote lab {},{}", task.getProjectId(), - uploadFile.getFilePath(), task.getServerIp(), ex.getLocalizedMessage()); - } - } - - private boolean stopCompatibilityTesting(CloudLabTestTask task) { - List commandList = new ArrayList<>(); - commandList.add("pid=$(pgrep -f sh\\ compatibility_testing.sh|head -1);sub_pids=$(pgrep -P ${pid});for " - + "subpid" + " " + "in ${sub_pids};do kill -9 \"${subpid}\" ;done;kill -9 \"${pid}\" "); - try { - SSHUtil.sshExecCmd(task.getServerIp(), task.getServerPort(), task.getServerUser(), - task.getServerPassword(), commandList); - - return true; - } catch (SshErrorException ex) { - log.error("Failed to execute task {},{}", task.getProjectId(), ex.getLocalizedMessage()); - return false; - } - } - - private boolean stopPerformanceTesting(CloudLabTestTask task) { - try { - String result = performanceApiClient.stopPerformanceTest(createURI(task.getPerformanceService()), - task.getProjectId()); - log.info("Task {} request {} stop performance test, return {}", task.getProjectId(), - task.getPerformanceService(), result); - JSONObject resultJson = JSONObject.parseObject(result); - return resultJson.containsKey("code") && "0000".equals(resultJson.getString("code")); - } catch (URISyntaxException ex) { - log.error("Failed to stop performance test {}", ex.getLocalizedMessage()); - return false; - } - } - - /** - * upload file - * - * @param file file - * @return response - */ - public String uploadFile(MultipartFile file) { - String fileId = null; - try { - fileId = fastDfsClient.uploadFile(file); - } catch (IOException ex) { - log.error(ex.getLocalizedMessage()); - } - return fileId; - } - - private URI createURI(String param) throws URISyntaxException { - ToolUtil.checkParameter(param); - return new URI(String.format(Locale.ROOT, - systemParams.getPerformanceTestUrl(), param)); - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/TarFileService.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/TarFileService.java deleted file mode 100644 index 95e1d89ad0bf17d912a2ed1b93222624e76482a9..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/TarFileService.java +++ /dev/null @@ -1,387 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.service; - -import com.huawei.ic.openlab.cloudtest.entity.CloudLabTestTask; -import com.huawei.ic.openlab.cloudtest.entity.SystemParams; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.CompatibilityTestResult; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.ParseFunctionTestLog; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.ParseInfoLog; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.ParseSafetyFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseClamLog; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseCpuComFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseCpuPerFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseCveCheckFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseDiskComFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseDiskPerFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseMemComFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseMemPerFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseNetComFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseNetPerFiles; -import com.huawei.ic.openlab.cloudtest.entity.compatibilityfileanalysis.impl.ParseSoftwareLog; -import com.huawei.ic.openlab.cloudtest.util.Constants; -import com.huawei.ic.openlab.cloudtest.util.ToolUtil; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.zip.ZipInputStream; - -/** - * TarFileService - * - * @author kongcaizhi - * @since 2021-10-19 - */ -@Service -@Slf4j -public class TarFileService { - /** - * BUFFER - */ - public static final int BUFFER = 512; - - /** - * TOOBIG - */ - public static final int TOOBIG = 0X64000000; - - private final SystemParams systemParams; - - /** - * TarFileService - * - * @param systemParams system Params - */ - public TarFileService(SystemParams systemParams) { - this.systemParams = systemParams; - } - - /** - * validatedTarGzFileSize - * - * @param fileInputStream input stream - * @throws IOException exception - */ - public void validatedTarGzFileSize(InputStream fileInputStream) throws IOException { - long total = 0L; - try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(fileInputStream))) { - while (zis.getNextEntry() != null) { - total = getTarTotalSize(total, zis); - if (total > TOOBIG) { - throw new IllegalStateException(Constants.TAR_FILE_TOO_BIG); - } - } - } - } - - private long getTarTotalSize(long total, ZipInputStream zis) throws IOException { - int count; - byte[] data = new byte[BUFFER]; - long sum = total; - while ((count = zis.read(data, 0, BUFFER)) != -1) { - sum += count; - if (sum > TOOBIG) { - break; - } - } - return sum; - } - - /** - * validatedZipFileSize - * - * @param fileInputStream input stream - * @param language language - * @throws IOException exception - */ - public void validatedZipFileSize(InputStream fileInputStream, String language) throws IOException { - long total = 0L; - try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new BufferedInputStream(fileInputStream))) { - while (zis.getNextEntry() != null) { - total = getZipTotalSize(total, zis); - if (total > TOOBIG) { - throw new IllegalStateException( - Constants.TAR_FILE_TOO_BIG_MAP.getOrDefault(language, "CN")); - } - } - } - } - - private long getZipTotalSize(long total, ZipArchiveInputStream zis) throws IOException { - int count; - byte[] data = new byte[BUFFER]; - long sum = total; - while ((count = zis.read(data, 0, BUFFER)) != -1) { - sum += count; - if (sum > TOOBIG) { - break; - } - } - return sum; - } - - /** - * 应用启动、停止、异常测试用例 - * - * @param filePath file path - * @param testResultList test result list - * @throws IOException exception - */ - private void getTestCaseFromInfoLog(String filePath, List testResultList) - throws IOException { - byte[] infoLogFile = readSpecificFile(filePath, Constants.CompatibilityTarFile.INFO_LOG.getFileName()); - ParseInfoLog parseInfoLog = new ParseInfoLog(infoLogFile, ""); - Map infoResultMap = parseInfoLog.parseInfoLog(); - for (Map.Entry entry : infoResultMap.entrySet()) { - testResultList.add(entry.getValue()); - } - } - - /** - * 软件识别用例 - * - * @param filePath file path - * @param testResultList test result list - * @throws IOException exception - */ - private void getTestCaseFromProductName(String filePath, List testResultList) - throws IOException { - byte[] configFile = readSpecificFile(filePath, Constants.CompatibilityTarFile.CONFIGURE_INFO.getFileName()); - byte[] processFile = readSpecificFile(filePath, Constants.CompatibilityTarFile.PRODUCT_NAME.getFileName()); - ParseSoftwareLog softwareLog = new ParseSoftwareLog(configFile, processFile); - CompatibilityTestResult softwareResult = softwareLog.parseFiles(); - testResultList.add(softwareResult); - } - - /** - * 空载测试用例 - * - * @param filePath filePath - * @param testResultList testResultList - * @throws IOException IOException - */ - private void getTestCaseFromIdleFile(String filePath, List testResultList) - throws IOException { - // 空载测试CPU 用例 - Map beginCpuComFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.COMPATIBILITY_CPU_0.getFileName()); - Map endCpuComFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.COMPATIBILITY_CPU_1.getFileName()); - CompatibilityTestResult cpuComResult = new ParseCpuComFiles(beginCpuComFile, endCpuComFile).parseFiles(); - testResultList.add(cpuComResult); - - // 空载测试内存用例 - Map beginMemComFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.COMPATIBILITY_MEM_0.getFileName()); - Map endMemComFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.COMPATIBILITY_MEM_1.getFileName()); - CompatibilityTestResult memComResult = new ParseMemComFiles(beginMemComFile, endMemComFile).parseFiles(); - testResultList.add(memComResult); - - Map beginDiskComFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.COMPATIBILITY_DISK_0.getFileName()); - Map endDiskComFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.COMPATIBILITY_DISK_1.getFileName()); - CompatibilityTestResult diskComResult = new ParseDiskComFiles(beginDiskComFile, endDiskComFile).parseFiles(); - testResultList.add(diskComResult); - - Map beginNetComFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.COMPATIBILITY_NET_0.getFileName()); - Map endNetComFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.COMPATIBILITY_NET_1.getFileName()); - CompatibilityTestResult diskNetResult = new ParseNetComFiles(beginNetComFile, endNetComFile).parseFiles(); - testResultList.add(diskNetResult); - } - - /** - * 压力测试用例 - * - * @param filePath filePath - * @param testResultList testResultList - * @throws IOException exception - */ - private void getTestCaseFromPressureFile(String filePath, List testResultList) - throws IOException { - Map cpuPerFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.PERFORMANCE_CPU.getFileName()); - CompatibilityTestResult cpuPerTestResult = new ParseCpuPerFiles(cpuPerFile).parseFiles(); - testResultList.add(cpuPerTestResult); - Map memPerFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.PERFORMANCE_MEM.getFileName()); - CompatibilityTestResult memPerTestResult = new ParseMemPerFiles(memPerFile).parseFiles(); - testResultList.add(memPerTestResult); - - Map diskPerFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.PERFORMANCE_DISK.getFileName()); - CompatibilityTestResult diskPerTestResult = new ParseDiskPerFiles(diskPerFile).parseFiles(); - testResultList.add(diskPerTestResult); - Map netPerFile = getSpecificFile(filePath, - Constants.CompatibilityTarFile.PERFORMANCE_NET.getFileName()); - CompatibilityTestResult netPerTestResult = new ParseNetPerFiles(netPerFile).parseFiles(); - testResultList.add(netPerTestResult); - } - - private void getTestCaseFromSafeFile(List fileList, String filePath, - List testResultList) throws IOException { - // 端口扫描 - byte[] protocolFile = new byte[0]; - byte[] updFile = new byte[0]; - byte[] tcpFile = new byte[0]; - - for (String tarFileName : fileList) { - if (StringUtils.endsWith(tarFileName, "gnmap") - && StringUtils.containsIgnoreCase(tarFileName, "PROTOCOL")) { - protocolFile = readSpecificFile(filePath, tarFileName); - } else if (StringUtils.endsWith(tarFileName, "gnmap") - && StringUtils.containsIgnoreCase(tarFileName, "UDP" - )) { - updFile = readSpecificFile(filePath, tarFileName); - } else if (StringUtils.endsWith(tarFileName, "gnmap") - && StringUtils.containsIgnoreCase(tarFileName, "TCP" - )) { - tcpFile = readSpecificFile(filePath, tarFileName); - } else { - log.info("not contrast"); - } - } - testResultList.add(new ParseSafetyFiles(protocolFile, updFile, tcpFile).parseSafetyFile()); - } - - /** - * compatibility Result Analysis - * - * @param task task - * @param multipartFile multipartFile - * @return CompatibilityTestResult list - */ - public List compatibilityResultAnalysis(String multipartFile) { - - String logPath = multipartFile; - File file = FileUtils.getFile(logPath); - List testResultList = new ArrayList<>(); - try { - List fileList = getTarFileList(logPath); - getTestCaseFromInfoLog(logPath, testResultList); - - // 硬件识别用例 - CompatibilityTestResult testResult = - new CompatibilityTestResult(Constants.CompatibilityTestName.HARDWARE_SERVER.getTestName()); - testResult.setResult(Constants.TEST_PASSED); - testResult.setReason(Constants.HCS_DESC); - testResultList.add(testResult); - - getTestCaseFromProductName(logPath, testResultList); - getTestCaseFromIdleFile(logPath, testResultList); - getTestCaseFromPressureFile(logPath, testResultList); - getTestCaseFromSafeFile(fileList, logPath, testResultList); - - // 功能测试 - byte[] shellBytes = readSpecificFile(logPath, - Constants.CompatibilityTarFile.FUNCTION_SHELL_UNIT.getFileName()); - byte[] pytestBytes = readSpecificFile(logPath, - Constants.CompatibilityTarFile.FUNCTION_PY_TEST.getFileName()); - testResultList.add(new ParseFunctionTestLog(shellBytes, pytestBytes).parseFunctionTestLog()); - } catch (IOException ex) { - log.error("Error parsing tar file", ex); - } - return testResultList; - } - - /** - * get tar file list - * - * @param tarGzFile tar file - * @return file list - * @throws IOException exception - */ - public List getTarFileList(String tarGzFile) throws IOException { - ToolUtil.validPath(tarGzFile); - List fileList = new ArrayList<>(); - try (FileInputStream fin = new FileInputStream(tarGzFile); - BufferedInputStream in = new BufferedInputStream(fin); - GzipCompressorInputStream gzip = new GzipCompressorInputStream(in); - TarArchiveInputStream tarIn = new TarArchiveInputStream(gzip)) { - TarArchiveEntry entry; - while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { - fileList.add(entry.getName()); - } - } - return fileList; - } - - /** - * get specific file - * - * @param tarGzFile tar file - * @param fileName file name - * @return file map - * @throws IOException exception - */ - public Map getSpecificFile(String tarGzFile, String fileName) throws IOException { - ToolUtil.validPath(tarGzFile); - Map fileMap = new HashMap<>(); - try (FileInputStream fin = new FileInputStream(tarGzFile); - BufferedInputStream in = new BufferedInputStream(fin); - GzipCompressorInputStream gzip = new GzipCompressorInputStream(in); - TarArchiveInputStream tarIn = new TarArchiveInputStream(gzip)) { - TarArchiveEntry entry; - - while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { - if (StringUtils.startsWithIgnoreCase(entry.getName(), fileName)) { - byte[] fileBytes = new byte[(int) entry.getSize()]; - tarIn.read(fileBytes); - fileMap.put(entry.getName(), fileBytes); - } - } - } - return fileMap; - } - - /** - * read specific file - * - * @param tarGzFile tar file - * @param fileName file name - * @return byte [] - * @throws IOException exception - */ - public byte[] readSpecificFile(String tarGzFile, String fileName) throws IOException { - ToolUtil.validPath(tarGzFile); - byte[] fileBytes = new byte[0]; - try (FileInputStream fin = new FileInputStream(tarGzFile); - BufferedInputStream in = new BufferedInputStream(fin); - GzipCompressorInputStream gzip = new GzipCompressorInputStream(in); - TarArchiveInputStream tarIn = new TarArchiveInputStream(gzip)) { - TarArchiveEntry entry; - while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { - if (StringUtils.startsWithIgnoreCase(entry.getName(), fileName)) { - fileBytes = new byte[(int) entry.getSize()]; - tarIn.read(fileBytes); - } - } - } - return fileBytes; - } -} \ No newline at end of file diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/TaskDelayMapService.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/TaskDelayMapService.java deleted file mode 100644 index e59911ce0241570cce66ecf1b8aaa64abc01aa14..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/service/TaskDelayMapService.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.service; - -import com.huawei.ic.openlab.cloudtest.common.exception.SshErrorException; -import com.huawei.ic.openlab.cloudtest.dao.CloudLabTestTaskDao; -import com.huawei.ic.openlab.cloudtest.entity.CloudLabTestTask; -import com.huawei.ic.openlab.cloudtest.entity.MqsMessage; -import com.huawei.ic.openlab.cloudtest.entity.SystemParams; -import com.huawei.ic.openlab.cloudtest.entity.TaskDelay; -import com.huawei.ic.openlab.cloudtest.util.Constants; -import com.huawei.ic.openlab.cloudtest.util.ToolUtil; -import com.huawei.ic.openlab.cloudtest.util.sshclient.SSHUtil; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * TaskDelayMapService - * - * @author kongcaizhi - * @since 2021-10-19 - */ -@Slf4j -public class TaskDelayMapService { - private final Map taskMap = new HashMap<>(); - private final CloudLabTestTaskDao dao; - private final SystemParams systemParams; - - /** - * construction function - * - * @param dao dao - * @param systemParams system parameter - */ - public TaskDelayMapService(CloudLabTestTaskDao dao, SystemParams systemParams) { - this.dao = dao; - this.systemParams = systemParams; - } - - /** - * add task - * - * @param projectId project id - * @param currentStatus status - */ - public void addTask(String projectId, String currentStatus) { - TaskDelay taskDelay; - if (taskMap.containsKey(projectId)) { - taskDelay = taskMap.get(projectId); - taskDelay.updateCurrentStatus(currentStatus); - } else { - taskDelay = new TaskDelay(projectId, ToolUtil.getMillionSeconds(), ToolUtil.getMillionSeconds(), - currentStatus); - } - taskMap.put(projectId, taskDelay); - } - - /** - * update task - * - * @param projectId project id - * @param currentStatus current status - * @return boolean - */ - public boolean updateTask(String projectId, String currentStatus) { - if (taskMap.containsKey(projectId)) { - TaskDelay taskDelay = taskMap.get(projectId); - taskDelay.updateCurrentStatus(currentStatus); - taskMap.put(projectId, taskDelay); - return true; - } - return false; - } - - /** - * get task status - * - * @param projectId project id - * @return status - */ - public String getTaskStatus(String projectId) { - if (taskMap.containsKey(projectId)) { - taskMap.get(projectId); - return taskMap.get(projectId).getCurrentStatus(); - } - return StringUtils.EMPTY; - } - - /** - * remove task - * - * @param projectId project id - * @return remove task - */ - public boolean removeTask(String projectId) { - if (taskMap.containsKey(projectId)) { - taskMap.remove(projectId); - return true; - } - return false; - } - - /** - * expire task - * - */ - @Scheduled(cron = "0 0/5 * * * ?") - public void expireTask() { - TaskDelay task; - Iterator> iterator = taskMap.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - task = entry.getValue(); - if (task.isExpired()) { - iterator.remove(); - CloudLabTestTask testTask = dao.getTestTask(task.getProjectId()); - testTask.setProjectStatus(Constants.CLOUD_LAB_TEST_EXCEPTION); - testTask.setStatusDesc(Constants.TASK_EXPIRE_ESC_MAP.get(testTask.getTaskLanguage())); - testTask.setFinishTime(ToolUtil.getStandardTime()); - dao.updateProjectStatus(testTask.getProjectId(), testTask.getProjectStatus(), "", - testTask.getFinishTime(), testTask.getStatusDesc()); - sendExceptionMessage(testTask); - - // 删除服务器上的测试文件 - deleteTestFile(testTask); - log.error("Task {} status {}: wait for 60 minutes without any change in status, abnormal testing task", - task.getProjectId(), task.getCurrentStatus()); - } - } - } - - private void sendExceptionMessage(CloudLabTestTask task) { - MqsMessage message = new MqsMessage(); - message.setProjectId(task.getProjectId()); - message.setUserId(task.getUserId()); - message.setServerIp(task.getServerIp()); - message.setStatusTime(task.getFinishTime()); - message.setStatus(Constants.MQS_STATUS_EXCEPTION); - message.setStatusDesc(task.getStatusDesc()); - } - - /** - * delete test file - * - * @param task task - */ - public void deleteTestFile(CloudLabTestTask task) { - try { - List commandList = new ArrayList<>(); - commandList.add("cd /home/compatibility_testing;find . -name \"*.sh\"| xargs rm -rf ; " - + "rm -rf compatibility_testing.tar.gz;"); - commandList.add("cd /home/function_testing; rm -rf shunit2-master.zip shunit2-master;"); - - SSHUtil.sshExecCmd(task.getServerIp(), task.getServerPort(), task.getServerUser(), - task.getServerPassword(), commandList); - } catch (SshErrorException ex) { - log.error("task {} in server {} failed in delete {}", task.getProjectId(), task.getServerIp(), - ex.getLocalizedMessage()); - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/AESUtil.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/AESUtil.java deleted file mode 100644 index a8ffc082d9676d1f9292d3cc4cbc235b150c24c6..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/AESUtil.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import java.nio.charset.StandardCharsets; -import java.security.GeneralSecurityException; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; -import java.util.Locale; -import java.util.Optional; - -import javax.annotation.PostConstruct; -import javax.crypto.Cipher; -import javax.crypto.KeyGenerator; -import javax.crypto.SecretKey; -import javax.crypto.spec.GCMParameterSpec; -import javax.crypto.spec.SecretKeySpec; - -/** - * AESUtil - * - * @author liuchunwang - * @since 2023-07-21 - */ -@Slf4j -@Component -public class AESUtil { - /** - * CRYPTO - */ - public static final Crypto CRYPTO = new Crypto(); - - private static final int PRIVATE_KEY_LENGTH = 256; - private static final int GCM_IV_LENGTH = 12; - private static final int GCM_TAG_LENGTH = 16; - - private final Environment environment; - - /** - * AESUtil - * - * @param environment env - */ - public AESUtil(Environment environment) { - this.environment = environment; - } - - /** - * Crypto - * - * @author liuchunwang - * @since 2023-07-21 - */ - @Data - public static class Crypto { - private String aesCipherMode; - private String aesDefaultKey; - private String aesDefaultAad; - } - - @PostConstruct - private void initCrypto() { - CRYPTO.aesCipherMode = environment.getRequiredProperty("crypto.aes-cipher-mode"); - CRYPTO.aesDefaultKey = environment.getRequiredProperty("crypto.aes-default-key"); - CRYPTO.aesDefaultAad = environment.getRequiredProperty("crypto.aes-default-aad"); - } - - /** - * get PrivateKey - * - * @return PrivateKey - */ - public static byte[] getPrivateKey() { - KeyGenerator keygen = null; - try { - keygen = KeyGenerator.getInstance("AES"); - } catch (NoSuchAlgorithmException e) { - log.error(e.getMessage(), e); - } - - if (keygen != null) { - keygen.init(AESUtil.PRIVATE_KEY_LENGTH); - return keygen.generateKey().getEncoded(); - } - return new byte[0]; - } - - /** - * gcmEncrypt - * - * @param input input - * @param key key - * @param aadStr aadStr - * @return String - */ - public static String gcmEncrypt(String input, String key, String aadStr) { - byte[] data = input.getBytes(StandardCharsets.UTF_8); - byte[] privateKey = hex2byte(key); - byte[] iv = genRandomBytes(); - int tagLength = GCM_TAG_LENGTH; - byte[] aad = StringUtils.isBlank(aadStr) ? null : aadStr.getBytes(StandardCharsets.UTF_8); - byte[] cipherData = Optional.ofNullable(gcmEncrypt(data, privateKey, iv, tagLength, aad)) - .orElseThrow(() -> new BaseException("Parsing exceptions")); - - byte[] ivLengthByteArray = int2ByteArray(iv.length); - byte[] tagLengthByteArray = int2ByteArray(tagLength); - int aadLength = aad == null ? 0 : aad.length; - byte[] addLengthByteArray = int2ByteArray(aadLength); - - byte[] result = new byte[12 + iv.length + aadLength + cipherData.length]; - System.arraycopy(ivLengthByteArray, 0, result, 0, 4); - System.arraycopy(tagLengthByteArray, 0, result, 4, 4); - System.arraycopy(addLengthByteArray, 0, result, 8, 4); - System.arraycopy(iv, 0, result, 12, iv.length); - if (aad != null) { - System.arraycopy(aad, 0, result, 12 + iv.length, aadLength); - } - System.arraycopy(cipherData, 0, result, 12 + iv.length + aadLength, cipherData.length); - return byte2Hex(result); - } - - /** - * gcmDecrypt - * - * @param input input - * @param key key - * @return String - */ - public static String gcmDecrypt(String input, String key) { - byte[] data = hex2byte(input); - byte[] paramLength = new byte[4]; - - System.arraycopy(data, 0, paramLength, 0, 4); - int ivLength = byteArray2Int(paramLength); - byte[] iv = new byte[ivLength]; - System.arraycopy(data, 12, iv, 0, ivLength); - System.arraycopy(data, 4, paramLength, 0, 4); - int tagLength = byteArray2Int(paramLength); - System.arraycopy(data, 8, paramLength, 0, 4); - - int aadLength = byteArray2Int(paramLength); - byte[] aad = aadLength == 0 ? null : new byte[aadLength]; - if (aad != null) { - System.arraycopy(data, 12 + ivLength, aad, 0, aadLength); - } - - byte[] cipherData = new byte[data.length - ivLength - aadLength - 12]; - System.arraycopy(data, 12 + ivLength + aadLength, cipherData, 0, cipherData.length); - - byte[] privateKey = hex2byte(key); - return new String(gcmDecrypt(cipherData, privateKey, iv, tagLength, aad)); - } - - private static byte[] gcmEncrypt(byte[] data, byte[] privateKey, byte[] iv, int tagLength, byte[] aad) { - try { - Cipher cipher = cipher(privateKey, iv, tagLength, Cipher.ENCRYPT_MODE); - return handle(data, cipher, aad); - } catch (GeneralSecurityException e) { - throw new BaseException("Parsing exceptions"); - } - } - - private static byte[] gcmDecrypt(byte[] data, byte[] privateKey, byte[] iv, int tagLength, byte[] aad) { - try { - Cipher cipher = cipher(privateKey, iv, tagLength, Cipher.DECRYPT_MODE); - return handle(data, cipher, aad); - } catch (GeneralSecurityException e) { - throw new BaseException("Parsing exceptions"); - } - } - - private static Cipher cipher(byte[] privateKey, byte[] iv, int tagLength, int mode) - throws GeneralSecurityException { - Cipher cipher = Cipher.getInstance(CRYPTO.aesCipherMode); - SecretKey secretKey = new SecretKeySpec(privateKey, "AES"); - GCMParameterSpec parameterSpec = new GCMParameterSpec(tagLength * Byte.SIZE, iv); - cipher.init(mode, secretKey, parameterSpec); - return cipher; - } - - private static byte[] handle(byte[] data, Cipher cipher, byte[] aad) throws GeneralSecurityException { - if (aad != null) { - cipher.updateAAD(aad); - } - return cipher.doFinal(data); - } - - private static byte[] genRandomBytes() { - byte[] result = new byte[AESUtil.GCM_IV_LENGTH]; - SecureRandom rand = RandomUtil.getRandom(); - rand.nextBytes(result); - return result; - } - - private static byte[] int2ByteArray(int i) { - byte[] result = new byte[4]; - result[0] = (byte) ((i >> 24) & 0xFF); - result[1] = (byte) ((i >> 16) & 0xFF); - result[2] = (byte) ((i >> 8) & 0xFF); - result[3] = (byte) (i & 0xFF); - return result; - } - - private static int byteArray2Int(byte[] bytes) { - int result = 0; - for (int i = 0; i < 4; i++) { - int shift = (3 - i) * 8; - result += (bytes[i] & 0xFF) << shift; - } - return result; - } - - private static String byte2Hex(byte[] bytes) { - StringBuilder sb = new StringBuilder(16); - for (byte b : bytes) { - String hex = Integer.toHexString(b & 0xFF); - if (hex.length() < 2) { - sb.append(0); - } - sb.append(hex.toUpperCase(Locale.US)); - } - return sb.toString(); - } - - private static byte[] hex2byte(String hexString) { - byte[] bytes = new byte[hexString.length() / 2]; - for (int i = 0; i < hexString.length(); i += 2) { - bytes[i / 2] = - (byte) ((Character.digit(hexString.charAt(i), 16) << 4) - + Character.digit(hexString.charAt(i + 1), 16)); - } - return bytes; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/ApiCenterUtil.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/ApiCenterUtil.java deleted file mode 100644 index 637885bdfdd852a76d420bb2fd192a9329e714e5..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/ApiCenterUtil.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import com.huawei.ic.openlab.cloudtest.entity.SystemParams; - -import com.alibaba.fastjson.JSONObject; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.springframework.web.client.RestClientException; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; - -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; -import java.util.Objects; - -import javax.net.ssl.SSLContext; - -/** - * API网关的接口工具类 - * - * @author kongcaizhi - * @since 2022-10-24 - */ -@Slf4j -public class ApiCenterUtil { - private static final String VALIDATED_TOKEN_SUCCESS_CODE = "200"; - - private static HttpHeaders getBasicHttpHeaders(SystemParams systemParams) { - HttpHeaders headers = new HttpHeaders(); - headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); - headers.add("X-HW-ID", systemParams.getAppId()); - headers.add("X-HW-APPKEY", systemParams.getAppKey()); - return headers; - } - - private static RestTemplate restTemplate() throws KeyStoreException, NoSuchAlgorithmException, - KeyManagementException { - TrustStrategy acceptingTrustStrategy = (X509Certificate[] chain, String authType) -> true; - SSLContext sslContext = org.apache.http.ssl.SSLContexts.custom() - .loadTrustMaterial(null, acceptingTrustStrategy) - .build(); - SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext); - CloseableHttpClient httpClient = HttpClients.custom() - .setSSLSocketFactory(csf) - .build(); - - HttpComponentsClientHttpRequestFactory requestFactory = - new HttpComponentsClientHttpRequestFactory(); - requestFactory.setHttpClient(httpClient); - - return new RestTemplate(requestFactory); - } - - /** - * 向API网关验证token是否有效 - * - * @param systemParams 系统参数 - * @param token token - * @param taskDeviceId taskDeviceId - * @return boolean 是否有效 - */ - public static boolean validateToken(SystemParams systemParams, String token, String taskDeviceId) { - // 添加请求头 - HttpEntity request = new HttpEntity<>(null, getBasicHttpHeaders(systemParams)); - - // 发送GET请求 - try { - UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(systemParams.getKunpengServiceUrl()) - .queryParam("sessionId", token) - .queryParam("token", taskDeviceId) - .queryParam("tag", 1); - String requestUrl = builder.build().encode().toUriString(); - ResponseEntity response = restTemplate().exchange(requestUrl, HttpMethod.GET, request, - ValidatedResp.class); - - log.info("request apiwg check token return{}", JSONObject.toJSONString(response.getBody())); - if (response.getStatusCode().is2xxSuccessful() && response.hasBody() - && VALIDATED_TOKEN_SUCCESS_CODE.equals(Objects.requireNonNull(response.getBody()).getCode())) { - return true; - } else { - return false; - } - } catch (KeyStoreException | NoSuchAlgorithmException | KeyManagementException | RestClientException ex) { - log.error("request apiwg check taskDeviceId {} exception {}", taskDeviceId, - ex.getLocalizedMessage()); - return false; - } - } - - @Data - private static class ValidatedResp { - private String code; - private String msg; - private String data; - @JsonProperty("success") - private Boolean isSuccess; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/Constants.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/Constants.java deleted file mode 100644 index 7f36f996415288444da46a1d3dbaea49e9d3da16..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/Constants.java +++ /dev/null @@ -1,789 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -/** - * Constants - * - * @author kongcaizhi - * @since 2022-10-24 - */ -public final class Constants { - /** - * TEST_PASSED - */ - public static final String TEST_PASSED = "passed"; - - /** - * TEST_FAILED - */ - public static final String TEST_FAILED = "failed"; - - /** - * TEST_SKIPPED - */ - public static final String TEST_SKIPPED = "skipped"; - - /** - * CLOUD_LAB_TEST_SUBMIT - */ - public static final Integer CLOUD_LAB_TEST_SUBMIT = 0; - - /** - * CLOUD_LAB_TEST_START - */ - public static final Integer CLOUD_LAB_TEST_START = 1; - - /** - * CLOUD_LAB_TEST_FINISH - */ - public static final Integer CLOUD_LAB_TEST_FINISH = 2; - - /** - * CLOUD_LAB_TEST_EXCEPTION - */ - public static final Integer CLOUD_LAB_TEST_EXCEPTION = 3; - - /** - * CLOUD_LAB_TEST_HALT - */ - public static final Integer CLOUD_LAB_TEST_HALT = 4; - - /** - * TAR_FILE_TOO_BIG - */ - public static final String TAR_FILE_TOO_BIG = "上传的文件解压后不能超过100M,请重新上传."; - - /** - * FILE_TOO_BIG - */ - public static final String FILE_TOO_BIG = "上传的文件不能超过100M,请重新上传."; - - /** - * CONNECT_EXCEPTION_DESC - */ - public static final String CONNECT_EXCEPTION_DESC = "连接异常,请检查登录密码."; - - /** - * TASK_DUPLICATE - */ - public static final String TASK_DUPLICATE = "%s服务器正在测试中,请勿重复提交."; - - /** - * TEMPLATE_DECOMPRESSION_ERROR - */ - public static final String TEMPLATE_DECOMPRESSION_ERROR = "请下载模板,根据模板修改,上传修改后的压缩文件"; - - /** - * FILE_FAILED_VIRUS_SCAN - */ - public static final String FILE_FAILED_VIRUS_SCAN = "文件没有通过病毒扫描,请重新上传"; - - /** - * 正常 - */ - public static final String UPLOAD_FILE_STATUS_NORMAL = "00"; - - /** - * 已删除 - */ - public static final String UPLOAD_FILE_STATUS_DELETE = "01"; - - /** - * 安全检查不通过 - */ - public static final String UPLOAD_FILE_STATUS_SCAN_FAIL = "02"; - - /** - * 文件太大 - */ - public static final String UPLOAD_FILE_STATUS_TOO_BIG = "03"; - - /** - * 文件异常 - */ - public static final String UPLOAD_FILE_STATUS_EXCEPTION = "04"; - - /** - * 安全测试文件缺失或者读取失败 - */ - public static final String SAFETY_FILE_LOSS = "安全测试文件缺失或者读取失败"; - - /** - * SAFETY_FILE_LOSS_EN - */ - public static final String SAFETY_FILE_LOSS_EN = "The security test file is missing or cannot be read"; - - /** - * HCS_DESC - */ - public static final String HCS_DESC = "硬件包含鲲鹏芯片，符合要求"; - - /** - * PERFORMANCE_TEST_ID - */ - public static final String PERFORMANCE_TEST_ID = "Performance_Test_1"; - - /** - * TASK_DOES_NOT_EXIST - */ - public static final String TASK_DOES_NOT_EXIST = "任务不存在"; - - /** - * TASK_HAS_FINISHED - */ - public static final String TASK_HAS_FINISHED = "测试已结束"; - - /** - * FAILED_TO_STOP_TASK - */ - public static final String FAILED_TO_STOP_TASK = "停止测试任务失败"; - - /** - * PERFORMANCE_TESTING - */ - public static final String PERFORMANCE_TESTING = "性能测试中"; - - /** - * START_TEST_STRING - */ - public static final String START_TEST_STRING = "自动化测试采集工具开始执行"; - - /** - * START_TEST_STRING_EN - */ - public static final String START_TEST_STRING_EN = "The automatic test tool starts to run."; - - /** - * DEPENDENCY_INSTALL_FAILURE_DESC - */ - public static final String DEPENDENCY_INSTALL_FAILURE_DESC = "依赖软件安装失败"; - - /** - * MQS_STATUS_ONGOING - */ - public static final String MQS_STATUS_ONGOING = "ongoing"; - - /** - * MQS_STATUS_FINISH - */ - public static final String MQS_STATUS_FINISH = "finished"; - - /** - * MQS_STATUS_EXCEPTION - */ - public static final String MQS_STATUS_EXCEPTION = "exception"; - - /** - * 依赖软件安装 - */ - public static final String DEPENDENCY_INSTALL_DESC = "0-1"; - - /** - * 应用软件启动 - */ - public static final String APP_START_DESC = "0-2"; - - /** - * 应用软件停止 - */ - public static final String APP_STOP_DESC = "0-3"; - - /** - * 安全测试 - */ - public static final String SECURITY_TEST_DESC = "1"; - - /** - * 可靠性测试 - */ - public static final String RELIABILITY_TEST_DESC = "2"; - - /** - * 兼容性测试 - */ - public static final String COMPATIBILITY_TEST_DESC = "3"; - - /** - * 功能测试 - */ - public static final String FUNCTION_TEST_DESC = "4"; - - /** - * 性能测试 - */ - public static final String PERFORMANCE_TEST_DESC = "5"; - - /** - * 测试状态-中 - */ - public static final String TEST_PROCESSING = "0"; - - /** - * 测试状态-完成 - */ - public static final String TEST_FINISHED_CN = "1"; - - /** - * 测试状态-失败 - */ - public static final String TEST_FAILURE_CN = "2"; - - /** - * 测试状态-异常 - */ - public static final String TEST_EXCEPTION_CN = "异常"; - - /** - * T测试状态-正常 - */ - public static final String TEST_NORMAL_CN = "正常"; - - /** - * RELIABILITY_START_APP_FAIL_CODE - */ - public static final String RELIABILITY_START_APP_FAIL_CODE = "0911"; - - /** - * OS_PATTERN_CN - */ - public static final String OS_PATTERN_CN = "#\\d{4}-\\d{2}-\\d{2}\\s+(20|21|22|23|[0-1]\\d)" - + ":[0-5]\\d:[0-5]\\d#info#1#(当前的操作系统版本是|The current OS version is)(.*)(.|.)"; - - /** - * APPLICATION_NAMES - */ - public static final String APPLICATION_NAMES = "^application_names=(.+)"; - - /** - * START_APP_COMMANDS - */ - public static final String START_APP_COMMANDS = "^start_app_commands=(.+)"; - - /** - * STOP_APP_COMMANDS - */ - public static final String STOP_APP_COMMANDS = "^stop_app_commands=(.+)"; - - /** - * CLAM_RESULT_PATTERN - */ - public static final String CLAM_RESULT_PATTERN = "Infected files: (.+)"; - - /** - * 功能测试内存 SHELL_UNIT_RUN_PATTERN - */ - public static final String SHELL_UNIT_RUN_PATTERN = "^Ran \\\u001B\\[1;36m(.*)\\\u001B\\[0m test"; - - /** - * 功能测试内存 SHELL_UNIT_TEST_PATTERN - */ - public static final String SHELL_UNIT_TEST_PATTERN = "^-+ Test file name : (.*).sh -+"; - - /** - * 功能测试内存 SHELL_UNIT_FAILED_PATTERN - */ - public static final String SHELL_UNIT_FAILED_PATTERN = "^\\\u001B\\[1;31mFAILED\\\u001B\\[0m \\(\\\u001B\\[1;" - + "31mfailures=(.*)\\\u001B\\[0m\\)"; - - /** - * PYTEST_SUCCESS_PATTERN - */ - public static final String PYTEST_SUCCESS_PATTERN = "^=+ (.*) passed in .* =+"; - - /** - * PYTEST_RUN_PATTERN - */ - public static final String PYTEST_RUN_PATTERN = "^-+ Test file name : (.*).py -+"; - - /** - * PYTEST_FAILED_PATTERN - */ - public static final String PYTEST_FAILED_PATTERN = "^=+ (.*) failed, (.*) passed in .*=+"; - - /** - * APP_PACKAGE_TEST_TOPIC - */ - public static final String APP_PACKAGE_TEST_TOPIC = "App_Package_Test"; - - /** - * SECURITY_SCAN_TAG - */ - public static final String SECURITY_SCAN_TAG = "Security_Scan"; - - /** - * INFO_TEST_CASE_RESULT_MAP - */ - public static final Map INFO_TEST_CASE_RESULT_MAP = - Collections.unmodifiableMap(new HashMap() { - { - put("FAILED_TO_START_APP", new InfoTestCaseResult( - TEST_FAILED, "软件启动失败.", "Failed to start the software.")); - put("SUCCESS_TO_START_APP", new InfoTestCaseResult( - TEST_PASSED, "", "")); - put("FAILED_TO_RELIABLE_TEST", new InfoTestCaseResult( - TEST_FAILED, "强制杀死进程,启动失败,异常测试失败.", - "After the process was forcibly killed," - + " the process failed to be restarted, and exception test failed.")); - put("SUCCESS_TO_RELIABLE_TEST", new InfoTestCaseResult( - TEST_PASSED, "", "")); - put("FAILED_TO_STOP_APP", new InfoTestCaseResult( - TEST_FAILED, "软件停止失败.", "Failed to stop the software.")); - put("SUCCESS_TO_STOP_APP", new InfoTestCaseResult( - TEST_PASSED, "", "")); - put("FILE_NOT_EXIT", new InfoTestCaseResult( - TEST_FAILED, "测试结果中没有兼容性测试工具的日志文件.", - "The compressed file package does not contain" - + " the log file of the compatibility test tool.")); - put("NO_START_APP", new InfoTestCaseResult(TEST_FAILED, "没有执行软件启动.", - "You have no started the software.")); - put("NO_STOP_APP", new InfoTestCaseResult(TEST_FAILED, "没有执行软件停止.", - "You have no stop the software.")); - put("NO_RELIABLE_TEST", new InfoTestCaseResult(TEST_FAILED, "没有执行可靠性测试.", - "You have no conducted the reliability test.")); - put("UNZIP_ERROR", new InfoTestCaseResult(TEST_FAILED, "测试结果解压错误.", - "An error occurred when decompressing the log file " - + "of the test tool from the uploaded file package.")); - put("READ_ERROR", new InfoTestCaseResult(TEST_FAILED, "测试结果读取错误.", - "An error occurred when reading the log file" - + " of the test tool from the uploaded file package.")); - } - }); - - /** - * SOFTWARE_COMPARE_DESC_LIST - */ - public static final List SOFTWARE_COMPARE_DESC_LIST = Collections.unmodifiableList( - Arrays.asList("未匹配到相应软件：%s", "兼容性测试配置文件填写的应用软件为：%s", - "兼容性测试配置文件应用软件名称为空", "没有找到进程快照文件", "解析配置文件和软件进程栈信息出错")); - - /** - * SOFTWARE_COMPARE_DESC_EN_LIST - */ - public static final List SOFTWARE_COMPARE_DESC_EN_LIST = Collections.unmodifiableList( - Arrays.asList( - "No matching software %s found.", "The application software recorded in the compatibility test " - + "configuration file is %s", - "The name of the application software in the compatibility test configuration file is %s.", - "Cannot find the process snapshot file in the uploaded file package.", - "An error occurred when reading the process snapshot file in the uploaded file package.")); - - /** - * COMPATIBILITY_DESC_LIST - */ - public static final List COMPATIBILITY_DESC_LIST = Collections.unmodifiableList( - Arrays.asList( - "被测试软件启动前采集%1$s资源利用率与被测试软件停止后%1$s利用率之间的波动为%2$,.2f %%.", - "被测试软件启动前采集%1$s资源利用率与被测试软件停止后%1$s利用率之间的波动为%2$,.2f %%,%3$s 1.00 %% .", - "被测试软件启动前采集%1$s资源利用率与被测试软件停止后%1$s资源利用率文件解析出错.", - "被测试软件启动前采集%1$s资源利用率与被测试软件停止后%1$s资源利用率文件缺失.", - "根据端口扫描的结果,结合软件端口矩阵,合适是否存在未知端口,并澄清原因.", "%s协议扫描结果如下：", - "被测试软件启动前采集的网卡资源%1$s数据与被测软件停止后采集网卡资源%1$s数据之间的波动为%2$," - + ".2f %%,%s测试前波动的1.00%%.", - "被测试软件启动前采集的网卡资源%1$s数据与被测软件停止后采集网卡资源%1$s数据之间的波动为%2$,.2f %%.")); - - /** - * COMPATIBILITY_DESC_EN_LIST - */ - public static final List COMPATIBILITY_DESC_EN_LIST = Collections.unmodifiableList( - Arrays.asList( - "The change rate of the %s usage collected before the tested software was started " - + "and after the target software was stopped is %.2f %%.", - "The change rate of the %s usage collected before the tested software was started " - + "and after the target software was stopped is %.2f %%, which is %s than 1.00%%.", - "An error occurred when parsing the resource files related to %s," - + "before the tested software was started and after the software was stopped.", - "An error occurred when reading the resource files related to %s," - + "before the tested software was started and after the software was stopped.", - "Check whether unknown ports are opened according to the port scanning " - + "result and the tested software port matrix, and locate the cause.", - "The scanning result of the %s protocols is as follows:", - "The data %s by the NIC fluctuates by %.2f %% before the tested software was " - + "started and after the tested software was stopped," - + " which is%s than 1.00%% before the test.", - "The data %s by the NIC fluctuates by %.2f %% before the tested software was " - + "started and after the software was stopped." - )); - - /** - * PERFORMANCE_DESC_LIST - */ - public static final List PERFORMANCE_DESC_LIST = Collections.unmodifiableList( - Arrays.asList( - "压力测试期间%s 资源波动值为%.2f %%,%s 5.00%%.", "后续可结合实际业务澄清波动合理性", - "压力测试期间%s 资源文件读取出错.", "未执行压力测试无 %s 资源信息", - "压力测试期间网卡%s数据波动是%.2f %%.", "压力测试期间网卡%s数据波动是%.2f %%,%s 5.00%%." - )); - - /** - * PERFORMANCE_DESC_EN_LIST - */ - public static final List PERFORMANCE_DESC_EN_LIST = Collections.unmodifiableList( - Arrays.asList( - "During the pressure test, the fluctuation value of %s is %.2f %%, which is %s than 5.00%%.", - "The cause of the fluctuation can be clarified based on actual services.", - "An error occurred when parsing the %s resource file during the pressure test.", - "The %s resource file is missing during the pressure test.", - "During the pressure test, the fluctuation value of data %s by the NIC is %.2f %%.", - "During the pressure test, the fluctuation value of data %s" - + " by the NIC is %.2f %%, which is %s than 5.00%%." - )); - - /** - * LOG_WHITE_RULES - */ - public static final Map LOG_WHITE_RULES = Collections.unmodifiableMap( - new HashMap() { - { - put("FAILED_TO_START_APP", Pattern.compile( - "#\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}#.*(#5)?#" - + "(检查到业务应用启动不成功|" - + "Failed to start the application (.*). Check the startup script.)")); - put("SUCCESS_TO_START_APP", Pattern.compile( - "#\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}#.*(#5)?#(业务应用(" - + ".*)(启动)?完成|" + "Succeeded in starting the application (.*))")); - put("FAILED_TO_RELIABLE_TEST", Pattern.compile( - "#\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}#.*(#8)?#" - + "(可靠性测试，业务应用(.*)启动失败，可靠性测试失败|" - + "可靠性测试，执行强制杀死进程(.*)报错，可靠性测试失败|" - + "可靠性测试前，业务应用(.*)已停止，可靠性测试失败|" - + "(.*)The reliability test failed.)")); - put("SUCCESS_TO_RELIABLE_TEST", Pattern.compile( - "#\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}#.*(#8)?#" - + "(可靠性测试，业务应用(.*)启动完成.可靠性测试成功|" - + "(.*)The reliability test is successful.)")); - put("FAILED_TO_STOP_APP", Pattern.compile( - "#\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}#.*(#9)?#" - + "(检查到应用程序还在启动，且用户未能停止应用|" - + "Failed to stop the service application (.*). Check the stop script.)")); - put("SUCCESS_TO_STOP_APP", Pattern.compile( - "#\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}#.*(#9)?#(进程(.*)" - + "不存在|The (.*) process does not exist.)")); - } - }); - - /** - * CONNECT_EXCEPTION_DESC_MAP - */ - public static final Map CONNECT_EXCEPTION_DESC_MAP = Collections.unmodifiableMap( - new HashMap() { - { - put("CN", CONNECT_EXCEPTION_DESC); - put("EN", "Connection exception. Check whether the login account and password are correct."); - } - }); - - /** - * TASK_DUPLICATE_MAP - */ - public static final Map TASK_DUPLICATE_MAP = Collections.unmodifiableMap( - new HashMap() { - { - put("CN", TASK_DUPLICATE); - put("EN", "The test is proceeding in %s. Please do not submit repeatedly."); - } - }); - - /** - * TASK_DUPLICATE_MAP - */ - public static final Map TAR_FILE_TOO_BIG_MAP = Collections.unmodifiableMap( - new HashMap() { - { - put("CN", TAR_FILE_TOO_BIG); - put("EN", "The uploaded file cannot exceed 100MB after decompression. " - + "Modify the file and upload it again."); - } - }); - - /** - * FAILED_ENCRYPT_MAP - */ - public static final Map FAILED_ENCRYPT_MAP = Collections.unmodifiableMap( - new HashMap() { - { - put("CN", "加密参数失败."); - put("EN", "Failed to encrypt the parameter."); - } - }); - - /** - * FAILED_ENCRYPT_MAP - */ - public static final Map TEMPLATE_DECOMPRESSION_ERROR_MAP = - Collections.unmodifiableMap(new HashMap() { - { - put("CN", TEMPLATE_DECOMPRESSION_ERROR); - put("EN", "Download the template, modify the file according to the template, " - + "and upload the compressed file after modification."); - } - }); - - /** - * FILE_FAILED_VIRUS_SCAN_MAP - */ - public static final Map FILE_FAILED_VIRUS_SCAN_MAP = - Collections.unmodifiableMap(new HashMap() { - { - put("CN", FILE_FAILED_VIRUS_SCAN); - put("EN", "The file failed the antivirus scan. Modify the file and upload it again."); - } - }); - - /** - * FILE_TOO_BIG_MAP - */ - public static final Map FILE_TOO_BIG_MAP = Collections.unmodifiableMap( - new HashMap() { - { - put("CN", FILE_TOO_BIG); - put("EN", "The uploaded file cannot exceed 100MB after decompression."); - } - }); - - /** - * FILE_TYPE_ERROR_MAP - */ - public static final Map FILE_TYPE_ERROR_MAP = Collections.unmodifiableMap( - new HashMap() { - { - put("CN", "文件解析出错."); - put("EN", "An error occurred when parsing the file."); - } - }); - - /** - * UPLOAD_FILE_STATUS_NORMAL_MAP - */ - public static final Map UPLOAD_FILE_STATUS_NORMAL_MAP = Collections.unmodifiableMap( - new HashMap() { - { - put("CN", "00"); - put("EN", "00"); - } - }); - - /** - * DEPENDENCY_INSTALL_FAILURE - */ - public static final Map DEPENDENCY_INSTALL_FAILURE = - Collections.unmodifiableMap(new HashMap() { - { - put("CN", "依赖软件安装异常."); - put("EN", "Dependency installation exception."); - } - }); - - /** - * APP_START_UP_FAILURE_DESC_MAP - */ - public static final Map APP_START_UP_FAILURE_DESC_MAP = - Collections.unmodifiableMap(new HashMap() { - { - put("CN", "待测试应用软件启动失败."); - put("EN", "Failed to start the application to be tested."); - } - }); - - /** - * APP_STOP_FAILURE_DESC_MAP - */ - public static final Map APP_STOP_FAILURE_DESC_MAP = - Collections.unmodifiableMap(new HashMap() { - { - put("CN", "待测试应用软件停止失败."); - put("EN", "Failed to stop the application to be tested."); - } - }); - - /** - * PERFORMANCE_FILE_MISSING_MAP - */ - public static final Map PERFORMANCE_FILE_MISSING_MAP = - Collections.unmodifiableMap(new HashMap() { - { - put("CN", "没有上传性能测试文件"); - put("EN", "No performance test file is uploaded."); - } - }); - - /** - * TASK_EXPIRE_ESC_MAP - */ - public static final Map TASK_EXPIRE_ESC_MAP = - Collections.unmodifiableMap(new HashMap() { - { - put("CN", "测试任务超时"); - put("EN", "The test task timed out without feedback."); - } - }); - - /** - * InfoTestCaseResult - * - * @author kongcaizhi - * @since 2021-10-19 - */ - @AllArgsConstructor - @Data - public static final class InfoTestCaseResult { - private final String testResult; - private final String testReason; - private final String testReasonEn; - } - - /** - * CompatibilityTarFile - * - * @author kongcaizhi - * @since 2021-10-19 - */ - public enum CompatibilityTarFile { - HARDWARE_INFO("data/hardware/hardware_info.log", 0), - INFO_LOG("data/others/info.log", 1), - CONFIGURE_INFO("data/compatibility_testing.conf", 2), - PRODUCT_NAME("data/product/product_name.log", 3), - COMPATIBILITY_CPU_0("data/test/compatiable/test_perf_cpu_0.log", 4), - COMPATIBILITY_CPU_1("data/test/compatiable/test_perf_cpu_1.log", 5), - COMPATIBILITY_MEM_0("data/test/compatiable/test_perf_mem_0.log", 6), - COMPATIBILITY_MEM_1("data/test/compatiable/test_perf_mem_1.log", 7), - COMPATIBILITY_DISK_0("data/test/compatiable/test_perf_disk_0.log", 8), - COMPATIBILITY_DISK_1("data/test/compatiable/test_perf_disk_1.log", 9), - COMPATIBILITY_NET_0("data/test/compatiable/test_perf_net_0.log", 10), - COMPATIBILITY_NET_1("data/test/compatiable/test_perf_net_1.log", 11), - PERFORMANCE_CPU("data/test/performance/test_perf_cpu_1.log", 12), - PERFORMANCE_MEM("data/test/performance/test_perf_mem_1.log", 13), - PERFORMANCE_DISK("data/test/performance/test_perf_disk_1.log", 14), - PERFORMANCE_NET("data/test/performance/test_perf_net_1.log", 15), - CLAM_LOG("data/test/safety/clam.log", 16), - FUNCTION_SHELL_UNIT("data/test/function/bash_test.log", 17), - FUNCTION_PY_TEST("data/test/function/python_test.log", 18), - CVE_CHECK_LOG("data/test/safety/cvecheck-result.json", 19); - - private final String fileName; - - CompatibilityTarFile(String fileName, int fileIndex) { - this.fileName = fileName; - } - - /** - * get File Name - * - * @return File Name - */ - public String getFileName() { - return this.fileName; - } - } - - /** - * CompatibilityTestName - * - * @author kongcaizhi - * @since 2021-10-19 - */ - public enum CompatibilityTestName { - HARDWARE_SERVER("Compatibility_Hardware_Server", 0), - SOFTWARE_NAME("Compatibility_Software_Name", 1), - APPLICATION_START("Compatibility_Application_Start", 2), - APPLICATION_STOP("Compatibility_Application_Stop", 3), - IDLE_CPU("Compatibility_Idle_Cpu", 4), - IDLE_MEM("Compatibility_Idle_Memory", 5), - IDLE_DISK("Compatibility_Idle_Disk", 6), - IDLE_NET("Compatibility_Idle_Network", 7), - PRESSURE_CPU("Reliability_Pressure_Cpu", 8), - PRESSURE_MEM("Reliability_Pressure_Memory", 9), - PRESSURE_DISK("Reliability_Pressure_Disk", 10), - PRESSURE_NET("Reliability_Pressure_Network", 11), - EXCEPTION_KILL("Reliability_Exception_Kill", 12), - SECURITY_PORT("Security_Base_Port", 13), - SECURITY_VIRUS("Security_Base_Virus", 14), - SECURITY_VULNERABLE("Security_Base_CveCheck", 15), - FUNCTION_TEST("Function_Test", 16); - - private final String testName; - - CompatibilityTestName(String testName, int testIndex) { - this.testName = testName; - } - - /** - * get Test Name - * - * @return Test Name - */ - public String getTestName() { - return this.testName; - } - } - - /** - * CompatibilityTestStep - * - * @author kongcaizhi - * @since 2021-10-19 - */ - public enum CompatibilityTestStep { - DEPENDENCY_INSTALL_SUCCESS("0210", "依赖软件安装完成"), - DEPENDENCY_INSTALL_FAILURE("0211", "依赖软件安装失败"), - APP_START_SUCCESS("0310", "应用软件启动完成"), - APP_START_FAILURE("0311", "应用软件启动失败"), - APP_STOP_SUCCESS("0320", "应用软件停止完成"), - APP_STOP_FAILURE("0321", "应用软件停止失败"), - COMPATIBILITY_TEST_SUCCESS("0410", "兼容性测试中"), - COM_START_SUCCESS("0510", "应用软件启动完成"), - COM_START_FAILURE("0511", "应用软件启动失败"), - PORT_TEST_SUCCESS("0610", "端口扫描完成"), - VIRUS_SCAN_SUCCESS("0620", "病毒扫描完成"), - VULNERABLE_SCAN_SUCCESS("0630", "漏洞扫描完成"), - EXCEPTION_TEST_FINISH("0810", "可靠性测试完成"), - COMPATIBILITY_TEST_FINISH("0910", "兼容性测试结束"), - FUN_TEST_FILE_EXIST_ERROR("1011", "测试失败-功能测试文件不存在"), - FUN_TEST_FILE_DIR_ERROR("1012", "测试失败-功能测试文件function_testing目录不存在"), - FUN_TEST_PYTEST_ERROR("1013", "测试失败-pytest安装失败"); - - private final String stepIndex; - - CompatibilityTestStep(String stepIndex, String stepDesc) { - this.stepIndex = stepIndex; - } - - /** - * get Step Index - * - * @return stepIndex - */ - public String getStepIndex() { - return this.stepIndex; - } - - /** - * fromStepIndex - * - * @param stepIndex stepIndex - * @return CompatibilityTestStep - */ - public static CompatibilityTestStep fromStepIndex(String stepIndex) { - CompatibilityTestStep compatibilityTestStep = null; - for (CompatibilityTestStep step : CompatibilityTestStep.values()) { - if (step.getStepIndex().equals(stepIndex)) { - compatibilityTestStep = step; - break; - } - } - return compatibilityTestStep; - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/DgCodeServerClient.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/DgCodeServerClient.java deleted file mode 100644 index d0a8e127f16328b4789ad106c194c72044fe5797..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/DgCodeServerClient.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import org.springframework.cloud.openfeign.FeignClient; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestParam; - -/** - * DGCodeServer nginx动态路由增加和删除taskDeviceId - * - * @author kongcaizhi - * @since 2023-03-23 - */ -@FeignClient(url = "${dgCodeServerUrl}", name = "dgCodeServer") -public interface DgCodeServerClient { - /** - * 路由增加taskDeviceId - * - * @param taskDeviceId 服务器信息 - * @param internalIp 服务器信息 - */ - @GetMapping("/add-task-device-id") - void addTaskDeviceId(@RequestParam("taskDeviceId") String taskDeviceId, - @RequestParam("internalIp") String internalIp); - - /** - * 路由删除taskDeviceId - * - * @param taskDeviceId taskdeviceId - */ - @GetMapping("/delete-task-device-id") - void deleteTaskDeviceId(@RequestParam("taskDeviceId") String taskDeviceId); -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/FileUtil.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/FileUtil.java deleted file mode 100644 index 0fc1a9171bff778e7c925d4a52f36a60b048ce83..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/FileUtil.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.fileupload.FileItemFactory; -import org.apache.commons.fileupload.disk.DiskFileItemFactory; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.multipart.commons.CommonsMultipartFile; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.OutputStream; - -/** - * FileUtil - * - * @author kongcaizhi - * @since 2023-03-23 - */ -@Slf4j -public class FileUtil { - /** - * fileToMultipartFile - * - * @param fileName fileName - * @return MultipartFile - */ - public static MultipartFile fileToMultipartFile(String fileName) { - FileItem fileItem = createFileItem(new File(fileName)); - return new CommonsMultipartFile(fileItem); - } - - private static FileItem createFileItem(File file) { - FileItemFactory factory = new DiskFileItemFactory(16, null); - FileItem item = factory.createItem("textField", "text/plain", true, file.getName()); - int bytesRead = 0; - byte[] buffer = new byte[8192]; - try (FileInputStream fis = new FileInputStream(file); - OutputStream os = item.getOutputStream()) { - while ((bytesRead = fis.read(buffer, 0, 8192)) != -1) { - os.write(buffer, 0, bytesRead); - } - } catch (IOException e) { - log.error("Exception in transforming File to multipartFile {}", e.getLocalizedMessage()); - } - return item; - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/MQSUtil.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/MQSUtil.java deleted file mode 100644 index 3c7930ae193222b921480547d11811ec0b2ab7d8..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/MQSUtil.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import com.huawei.ic.openlab.cloudtest.entity.MqsMessage; -import com.huawei.ic.openlab.cloudtest.entity.SystemParams; - -import com.alibaba.fastjson.JSONObject; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.springframework.web.client.RestTemplate; - -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; - -import javax.net.ssl.SSLContext; - -/** - * MQSUtil - * - * @author kongcaizhi - * @since 2023-03-23 - */ -@Slf4j -public class MQSUtil { - -} - diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/NormalResp.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/NormalResp.java deleted file mode 100644 index 8a204f31d35c36a62a7001cd1b4f493f70aed18e..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/NormalResp.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; - -import lombok.Builder; -import lombok.Data; - -/** - * NormalResp - * - * @author kongcaizhi - * @since 2021-03-23 - */ -@Data -@Builder -public class NormalResp { - private String code; - private String msg; - private String data; - - /** - * toJSONString - * - * @return String - */ - public String toJSONString() { - return JSON.toJSONString(this, SerializerFeature.WriteMapNullValue); - } - - /** - * ok - * - * @return String - */ - public static String ok() { - return NormalResp.builder().code("0000").msg("Success").build().toJSONString(); - } - - /** - * ok - * - * @param data data - * @return String - */ - public static String ok(String data) { - return NormalResp.builder().code("0000").data(data).msg("Success").build().toJSONString(); - } - - /** - * badRequest - * - * @param data data - * @return String - */ - public static String badRequest(String data) { - return NormalResp.builder().code("4001").data(data).msg("Bad Request").build().toJSONString(); - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/PBKDF2Util.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/PBKDF2Util.java deleted file mode 100644 index 3eaeb2367eacf5ecbab4656f5464ad297a9b7b2d..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/PBKDF2Util.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; - -import lombok.extern.slf4j.Slf4j; - -import java.io.UnsupportedEncodingException; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.security.NoSuchAlgorithmException; -import java.security.spec.InvalidKeySpecException; - -import javax.crypto.SecretKey; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.PBEKeySpec; - -/** - * Pbkdf2Util - * - * @since 2022-11-08 - * @author kongcaizhi - */ -@Slf4j -public class PBKDF2Util { - private static final String DEFAULT_CIPHER_AND_PADDING = "PBKDF2WithHmacSHA256"; - private static final int ITERATION_COUNT = 10000000; - private static final int KEY_LENGTH = 256; - private static final int COMPONENT_LENGTH = 48; - - private static byte[] digest(char[] password, String salt) throws NoSuchAlgorithmException, - InvalidKeySpecException { - SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(DEFAULT_CIPHER_AND_PADDING); - PBEKeySpec pbeKeySpec = new PBEKeySpec(password, salt.getBytes(Charset.defaultCharset()), ITERATION_COUNT, - KEY_LENGTH); - SecretKey secretKey = keyFactory.generateSecret(pbeKeySpec); - return secretKey.getEncoded(); - } - - /** - * pbkdf2ForRootKey - * - * @param salt salt - * @param components components - * @return String - */ - public static String pbkdf2ForRootKey(String salt, String... components) { - char[] component = new char[COMPONENT_LENGTH]; - for (int i = 0; i < COMPONENT_LENGTH; i++) { - component[i] = 0; - for (String s : components) { - char[] temp = s.toCharArray(); - component[i] ^= temp[i]; - } - } - - try { - return new String(PBKDF2Util.digest(component, salt), StandardCharsets.UTF_8.name()); - } catch (UnsupportedEncodingException | NoSuchAlgorithmException | InvalidKeySpecException e) { - log.error(e.getMessage(), e); - throw new BaseException(e.getMessage()); - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/PerformanceApiClient.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/PerformanceApiClient.java deleted file mode 100644 index c00baeb8638bf9cb12e343d15bca647749f20a39..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/PerformanceApiClient.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import org.springframework.cloud.openfeign.FeignClient; -import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RequestPart; -import org.springframework.web.multipart.MultipartFile; - -import java.net.URI; - -/** - * NormalResp - * - * @author kongcaizhi - * @since 2021-03-23 - */ -@FeignClient(url = "EMPTY", name = "PerformanceTest") -public interface PerformanceApiClient { - /** - * set Performance Test - * - * @param uri uri - * @param projectId projectId - * @param serverIp serverIp - * @param isTimeLimit isTimeLimit - * @param multipartFile multipartFile - * @return result - */ - @PostMapping( - value = "/{projectId}/performance-upload", - consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - String setPerformanceTest(URI uri, @PathVariable("projectId") String projectId, - @RequestParam("serverIp") String serverIp, - @RequestParam("timeLimit") boolean isTimeLimit, - @RequestPart(value = "file") MultipartFile multipartFile); - - /** - * stop Performance Test - * - * @param uri uri - * @param projectId projectId - * @return result - */ - @GetMapping("/{projectId}/stop-test") - String stopPerformanceTest(URI uri, @PathVariable("projectId") String projectId); -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/RandomUtil.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/RandomUtil.java deleted file mode 100644 index 491493f3fd12232e667054e5e04269c560790a72..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/RandomUtil.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; - -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; - -/** - * RandomUtil - * - * @author liuchunwang - * @since 2023-11-03 - */ -public class RandomUtil { - /** - * getRandom - * - * @return SecureRandom - */ - public static SecureRandom getRandom() { - try { - return SecureRandom.getInstanceStrong(); - } catch (NoSuchAlgorithmException e) { - throw new BaseException("make random number exception"); - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/TeeCodeServerClient.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/TeeCodeServerClient.java deleted file mode 100644 index 0d3ca09c4cc2b8554a9d7331898714c3ae79b5d6..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/TeeCodeServerClient.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import org.springframework.cloud.openfeign.FeignClient; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestParam; - -/** - * TeeCodeServer nginx动态路由增加和删除taskDeviceId - * - * @author kongcaizhi - * @since 2022-12-20 - */ -@FeignClient(url = "${teeCodeServerUrl}", name = "TeeCodeServer") -public interface TeeCodeServerClient { - /** - * 路由增加taskDeviceId - * - * @param taskDeviceId 服务器信息 - * @param internalIp 服务器信息 - */ - @GetMapping("/add-task-device-id") - void addTaskDeviceId(@RequestParam("taskDeviceId") String taskDeviceId, - @RequestParam("internalIp") String internalIp); - - /** - * 路由删除taskDeviceId - * - * @param taskDeviceId taskdeviceId - */ - @GetMapping("/delete-task-device-id") - void deleteTaskDeviceId(@RequestParam("taskDeviceId") String taskDeviceId); -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/ToolUtil.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/ToolUtil.java deleted file mode 100644 index d175240b14f5c0fd1b0333ad21e7dd02d7a6ee35..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/ToolUtil.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util; - -import com.huawei.ic.openlab.cloudtest.common.exception.BaseException; - -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * ToolUtil - * - * @author kongcaizhi - * @since 2021-03-23 - */ -public class ToolUtil { - private static final DateTimeFormatter STRING_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS"); - private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - private static final DateTimeFormatter STANDARD_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS"); - private static final String REGEX = "\\.\\.+|\\/[.|\\\\]\\/"; - private static final String REGEX_URL = "([0-9]+\\.){3}[0-9]+"; - - /** - * get Time Str - * - * @return Time Str - */ - public static String getTimeStr() { - return LocalDateTime.now().format(STRING_FORMATTER); - } - - /** - * get Standard Time - * - * @return Standard Time - */ - public static String getStandardTime() { - return LocalDateTime.now().format(STANDARD_FORMATTER); - } - - /** - * get data Time - * - * @return data Time - */ - public static String getDateStr() { - return LocalDateTime.now().format(DATE_FORMATTER); - } - - /** - * getMillionSeconds - * - * @return MillionSeconds - */ - public static Long getMillionSeconds() { - return System.currentTimeMillis(); - } - - /** - * validPath - * - * @param path path - */ - public static void validPath(String path) { - Pattern pattern = Pattern.compile(REGEX); - Matcher matcher = pattern.matcher(path); - if (matcher.find()) { - throw new BaseException("The File's path is illegal: " + path); - } - } - - /** - * checkParameter - * - * @param param param - */ - public static void checkParameter(String param) { - Pattern pattern = Pattern.compile(REGEX_URL); - Matcher matcher = pattern.matcher(param); - if (!matcher.find()) { - throw new BaseException("The URL's param is illegal: " + param); - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/fastdfs/FastDfsClient.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/fastdfs/FastDfsClient.java deleted file mode 100644 index 569a9ecbd39f9a28c78242ea5a5ae714d8c1282f..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/fastdfs/FastDfsClient.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util.fastdfs; - -import com.github.tobato.fastdfs.domain.fdfs.StorePath; -import com.github.tobato.fastdfs.service.FastFileStorageClient; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.commons.io.FilenameUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.multipart.MultipartFile; - -import java.io.IOException; - -/** - * FastDfsClient - * - * @author kongcaizhi - * @since 2021-10-19 - */ -@Slf4j -@Component -public class FastDfsClient { - @Autowired - private FastFileStorageClient storageClient; - - /** - * 上传文件 - * - * @param file 文件对象 - * @return 文件访问地址 - * @throws IOException IOException - */ - public String uploadFile(MultipartFile file) throws IOException { - StorePath storePath = storageClient.uploadFile(file.getInputStream(), file.getSize(), - FilenameUtils.getExtension(file.getOriginalFilename()), null); - return getResAccessUrl(storePath); - } - - private String getResAccessUrl(StorePath storePath) { - return storePath.getFullPath(); - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/sshclient/SFTPUtil.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/sshclient/SFTPUtil.java deleted file mode 100644 index 25c22124fbea86ad36966d2ec9546ec6a1575503..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/sshclient/SFTPUtil.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util.sshclient; - -import com.huawei.ic.openlab.cloudtest.entity.CloudLabTestTask; -import com.huawei.ic.openlab.cloudtest.entity.LabTestReq; - -import com.jcraft.jsch.Channel; -import com.jcraft.jsch.ChannelSftp; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.JSchException; -import com.jcraft.jsch.Session; -import com.jcraft.jsch.SftpException; - -import lombok.extern.slf4j.Slf4j; - -import java.io.InputStream; -import java.util.Properties; - -/** - * FastDfsClient - * - * @author kongcaizhi - * @since 2021-10-19 - */ -@Slf4j -public class SFTPUtil { - private static final Integer DEFAULT_TIMEOUT = 1000 * 60; - - /** - * upload file - * - * @param testReq testReq - * @param targetPath target path - * @param in input stream - * @throws JSchException JSchException - * @throws SftpException SftpException - */ - public static void uploadFile(LabTestReq testReq, - String targetPath, - InputStream in) throws JSchException, SftpException { - JSch jsch = new JSch(); - Session sshSession = jsch.getSession(testReq.getServerUser(), testReq.getServerIp(), testReq.getServerPort()); - sshSession.setPassword(testReq.getServerPassword()); - Properties config = new Properties(); - config.put("StrictHostKeyChecking", "no"); - sshSession.setConfig(config); - sshSession.connect(DEFAULT_TIMEOUT); - - Channel channel = sshSession.openChannel("sftp"); - channel.connect(); - ChannelSftp sftp = null; - if (!(channel instanceof ChannelSftp)) { - return; - } - try { - sftp = (ChannelSftp) channel; - sftp.put(in, targetPath); - } finally { - if (sftp != null) { - sftp.exit(); - } - } - } - - /** - * upload file - * - * @param task testReq - * @param targetPath target path - * @param in input stream - * @throws JSchException JSchException - * @throws SftpException SftpException - */ - public static void uploadFile(CloudLabTestTask task, - String targetPath, - InputStream in) throws JSchException, SftpException { - JSch jsch = new JSch(); - Session sshSession = jsch.getSession(task.getServerUser(), task.getServerIp(), task.getServerPort()); - sshSession.setPassword(task.getServerPassword()); - Properties config = new Properties(); - config.put("StrictHostKeyChecking", "no"); - sshSession.setConfig(config); - sshSession.connect(DEFAULT_TIMEOUT); - - Channel channel = sshSession.openChannel("sftp"); - channel.connect(); - ChannelSftp sftp = null; - if (!(channel instanceof ChannelSftp)) { - return; - } - try { - sftp = (ChannelSftp) channel; - sftp.put(in, targetPath); - } finally { - if (sftp != null) { - sftp.exit(); - } - } - } -} diff --git a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/sshclient/SSHUtil.java b/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/sshclient/SSHUtil.java deleted file mode 100644 index 8882cb4b0e69d1c3573c34a45dd637e8fccdad2b..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/java/com/huawei/ic/openlab/cloudtest/util/sshclient/SSHUtil.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2021-2023. All rights reserved. - */ - -package com.huawei.ic.openlab.cloudtest.util.sshclient; - -import com.huawei.ic.openlab.cloudtest.common.exception.SshErrorException; - -import lombok.extern.slf4j.Slf4j; - -import org.apache.sshd.client.SshClient; -import org.apache.sshd.client.channel.ChannelExec; -import org.apache.sshd.client.channel.ClientChannelEvent; -import org.apache.sshd.client.session.ClientSession; -import org.apache.sshd.common.util.io.output.NoCloseOutputStream; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * SSH 远程连接服务 - * - * @author kongcaizhi - * @since 2021-10-19 - */ -@Slf4j -public class SSHUtil { - private static final Integer DEFAULT_TIMEOUT = 1000 * 30; - private static final String DEFAULT_SHELL_CHARSET_NAME = "UTF-8"; - - /** - * 测试ssh连接 - * - * @param ip ip - * @param port port - * @param userName username - * @param passWord password - * @return boolean - * @throws SshErrorException exception - */ - public static boolean sshConnectTest(String ip, int port, String userName, String passWord) { - try (SshClient client = SshClient.setUpDefaultClient()) { - client.start(); - try (ClientSession session = client.connect(userName, ip, port).verify(DEFAULT_TIMEOUT).getSession()) { - session.addPasswordIdentity(passWord); - return session.auth().verify(DEFAULT_TIMEOUT).isSuccess(); - } finally { - client.stop(); - } - } catch (IOException e) { - log.error(e.getLocalizedMessage(), e); - } - return false; - } - - /** - * sshExecCmd - * - * @param ip ip - * @param port port - * @param userName username - * @param passWord password - * @param commandList command list - * @return string - * @throws SshErrorException exception - */ - public static String sshExecCmd(String ip, int port, String userName, String passWord, List commandList) - throws SshErrorException { - try (SshClient client = SshClient.setUpDefaultClient()) { - client.start(); - try (ClientSession session = client.connect(userName, ip, port).verify(DEFAULT_TIMEOUT).getSession()) { - return execCmd(passWord, session, commandList, DEFAULT_SHELL_CHARSET_NAME); - } finally { - client.stop(); - } - } catch (IOException e) { - log.error(e.getLocalizedMessage(), e); - throw new SshErrorException("SSH出现IO异常"); - } - } - - private static String execCmd(String passWord, ClientSession session, List commandList, String charsetName) - throws IOException, SshErrorException { - session.addPasswordIdentity(passWord); - if (session.auth().verify(DEFAULT_TIMEOUT).isFailure()) { - throw new SshErrorException("SSH 连接超时"); - } - try { - String response = ""; - for (String command : commandList) { - response = exec(session, command, charsetName); - } - return response; - } finally { - session.close(false); - } - } - - private static String exec(ClientSession session, String command, String charsetName) - throws IOException, SshErrorException { - try (ChannelExec channelExec = session.createExecChannel(command); - ByteArrayOutputStream out = new ByteArrayOutputStream()) { - channelExec.setOut(new NoCloseOutputStream(out)); - channelExec.setErr(new NoCloseOutputStream(out)); - if (!channelExec.open().verify(DEFAULT_TIMEOUT).isOpened()) { - throw new SshErrorException("SSH 连接异常"); - } - List list = new ArrayList<>(); - list.add(ClientChannelEvent.CLOSED); - channelExec.waitFor(list, DEFAULT_TIMEOUT); - channelExec.close(); - - String response = out.toString(charsetName); - String[] lines = response.split(System.lineSeparator()); - for (String line : lines) { - log.debug(line); - } - return response; - } - } -} diff --git a/component/CloudTest/src/main/resources/application.yml b/component/CloudTest/src/main/resources/application.yml deleted file mode 100644 index 34b6de0b540048b34c737b1df3c2d5ba1c0e360b..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/resources/application.yml +++ /dev/null @@ -1,30 +0,0 @@ -server: - port: 10037 - -scriptConfigCn: /data/software/cloudTest/script/compatibility_testing.conf -scriptConfigEn: /data/software/cloudTest/script/compatibility_testing_EN.conf -tempDir: /data/software/cloudTest/tmp -deployIp: 172.39.92.22 -uploadFileDir: /data/software/cloudTest/data -appId: com.huawei.icopenlab.website -appKey: ENC(avto39VXGCZudTEfU1DQjuou0INTVQpvA75GmdwZzIA=) -mqsTopic: T_OPENLAB_TEST_PROJECT_MESSAGE -mqsUrl: https://apigw-beta.huawei.com/api/mqs/message/beta -clamdIp : localhost -clamdPort : 3310 -performanceTestUrl : http://%s:8095/rest/v1/cloud-test/lab -appIdUat: com.huawei.icopenlab.website -appKeyUat: ENC(avto39VXGCZudTEfU1DQjuou0INTVQpvA75GmdwZzIA=) -mqsTopicUat: T_OPENLAB_TEST_PROJECT_MESSAGE -mqsUrlUat: https://mqs/message/beta -kunpengServiceUrl: https://kunpengcommunity/cloudbase/auth -teeCodeServerUrl : http://1111/code-server/ -dgCodeServerUrl: http://1111/code-server/ - -crypto: - aes-default-key: ENC(x2P5OjvjSqOMi9AGhKR70e1o5XLRd9XuUdfZPfFiL2VeQ5RTs498/f85N4moRzeBbr1to42juDgqnsuw8Wn9ykc9WrpdZ2AiwYdV09O9600=) - aes-default-aad: openlab - aes-cipher-mode: AES/GCM/PKCS5Padding -logging: - level: - root: debug \ No newline at end of file diff --git a/component/CloudTest/src/main/resources/fastjson.properties b/component/CloudTest/src/main/resources/fastjson.properties deleted file mode 100644 index f3c0788f4563b20a715c78d25f04902faa57426c..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/resources/fastjson.properties +++ /dev/null @@ -1 +0,0 @@ -fastjson.parser.safeMode=true \ No newline at end of file diff --git a/component/CloudTest/src/main/resources/key/EXTRUBjR/NmsaKOHM b/component/CloudTest/src/main/resources/key/EXTRUBjR/NmsaKOHM deleted file mode 100644 index 94e234935ba95ce0b06661b8c6df079e0353558d..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/resources/key/EXTRUBjR/NmsaKOHM +++ /dev/null @@ -1 +0,0 @@ -8BQTpsioA5ulNBnfY16hfL34mAsm1t98CQHd6h9EjLM0sgob \ No newline at end of file diff --git a/component/CloudTest/src/main/resources/key/STZQZzfs/PIeavG3r b/component/CloudTest/src/main/resources/key/STZQZzfs/PIeavG3r deleted file mode 100644 index 9f1b0fefdddf37ac11341521d97c93c12ad95154..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/resources/key/STZQZzfs/PIeavG3r +++ /dev/null @@ -1 +0,0 @@ -hyDu3TZOu2EcY61T5fo2VzdHOP4pp0wU4r1csz9Fne4wVmXt \ No newline at end of file diff --git a/component/CloudTest/src/main/resources/key/SW7HH3Up/59X8ARQZ b/component/CloudTest/src/main/resources/key/SW7HH3Up/59X8ARQZ deleted file mode 100644 index 52be9acfaf716af9c271aee02f88ba7949c750d2..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/resources/key/SW7HH3Up/59X8ARQZ +++ /dev/null @@ -1 +0,0 @@ -ijRxLKur4sztVzq4Ff7xqSgafxLGcMsG7HqV1H8bModYK5Cn \ No newline at end of file diff --git a/component/CloudTest/src/main/resources/logback-spring.xml b/component/CloudTest/src/main/resources/logback-spring.xml deleted file mode 100644 index d0aa28d2ebc33ac89cf137b45cdc769c08597207..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/resources/logback-spring.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - - - - - - - ${LOG_HOME}/cloudTest_%d{yyyy-MM-dd}.%i.log - - 10MB - - - 30 - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - UTF-8 - - - - - - - - - - - - - - \ No newline at end of file diff --git a/component/CloudTest/src/main/resources/mapper/CloudDevUseRecordMapper.xml b/component/CloudTest/src/main/resources/mapper/CloudDevUseRecordMapper.xml deleted file mode 100644 index 51315442769c7cbd3071b286dee3cc86ce283132..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/resources/mapper/CloudDevUseRecordMapper.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - INSERT INTO CLOUD_DEV_USE_RECORD( - REQUEST_TIME, - TASK_DEVICE_ID, - KP_TOKEN - )VALUE( - #{requestTime}, - #{taskDeviceId}, - #{kpToken} - ) - - \ No newline at end of file diff --git a/component/CloudTest/src/main/resources/mapper/CloudLabTestTaskMapper.xml b/component/CloudTest/src/main/resources/mapper/CloudLabTestTaskMapper.xml deleted file mode 100644 index 60ddf319766aee068f7a6c3dc7598b20c9939784..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/resources/mapper/CloudLabTestTaskMapper.xml +++ /dev/null @@ -1,266 +0,0 @@ - - - - - - - - - - - - - - - - - - - INSERT INTO CLOUD_LAB_TEST_TASK( - PROJECT_ID, - USER_ID, - SERVER_IP, - SERVER_PORT, - SERVER_USER, - SERVER_PASSWORD, - PROJECT_STATUS, - REQUEST_TIME, - START_TIME, - FINISH_TIME, - COMPATIBILITY_TEST, - RELIABILITY_TEST, - SECURITY_TEST, - FUNCTION_TEST, - PERFORMANCE_TEST, - STEP_STATUS, - CURRENT_STATUS, - FUNCTION_FILE, - PERFORMANCE_FILE, - RESULT_FILE, - TEST_SUMMARY, - TEST_DETAIL, - TASK_LANGUAGE - ) - VALUE( - #{projectId}, - #{data.userId}, - #{data.serverIp}, - #{data.serverPort}, - #{data.serverUser}, - #{data.serverPassword, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.CryptoTypeHandler}, - #{data.projectStatus}, - #{data.requestTime}, - #{data.startTime}, - #{data.finishTime}, - #{data.compatibilityTest}, - #{data.reliabilityTest}, - #{data.securityTest}, - #{data.functionTest}, - #{data.performanceTest}, - #{data.stepStatus, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - #{data.currentStatus}, - #{data.functionFile, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - #{data.performanceFile, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - #{data.resultFile, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - #{data.testSummary, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - #{data.testDetail, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - #{data.taskLanguage} - )ON DUPLICATE KEY UPDATE - SERVER_IP = #{data.serverIp}, - SERVER_PORT = #{data.serverPort}, - SERVER_USER = #{data.serverUser}, - SERVER_PASSWORD = - #{data.serverPassword, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.CryptoTypeHandler}, - PROJECT_STATUS = #{data.projectStatus}, - REQUEST_TIME = #{data.requestTime}, - COMPATIBILITY_TEST = #{data.compatibilityTest}, - RELIABILITY_TEST = #{data.reliabilityTest}, - SECURITY_TEST = #{data.securityTest}, - FUNCTION_TEST = #{data.functionTest}, - PERFORMANCE_TEST = #{data.performanceTest}, - TASK_LANGUAGE = #{data.taskLanguage}, - STEP_STATUS = null, - CURRENT_STATUS = null, - RESULT_FILE = null, - TEST_SUMMARY = null, - TEST_DETAIL = null, - SCRIPT_RESULT_CONFIG = null - - - - UPDATE CLOUD_LAB_TEST_TASK - SET - PROJECT_STATUS = #{status}, - STATUS_DESC = #{statusDesc}, - - START_TIME = #{startTime} - - - FINISH_TIME = #{finishTime} - - WHERE PROJECT_ID = #{projectId} - - - - - SELECT COUNT(*) FROM CLOUD_LAB_TEST_TASK WHERE PROJECT_ID = #{projectId} - - - - UPDATE CLOUD_LAB_TEST_TASK - SET - PERFORMANCE_SERVICE = #{ip}, - - FUNCTION_FILE = - #{data, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler} - - - PERFORMANCE_FILE = - #{data, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler} - - WHERE PROJECT_ID = #{projectId} - - - INSERT INTO CLOUD_LAB_TEST_TASK( - PROJECT_ID, - USER_ID, - PERFORMANCE_SERVICE, - - FUNCTION_FILE - - - PERFORMANCE_FILE - - )VALUE( - #{projectId}, - #{userId}, - #{ip}, - #{data, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler} - ) - - - - - - UPDATE CLOUD_LAB_TEST_TASK - SET - STEP_STATUS = #{data, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - TEST_BEGIN_TIME = - #{testBeginTime, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler} - WHERE PROJECT_ID = #{projectId} - - - - UPDATE CLOUD_LAB_TEST_TASK - SET - STEP_STATUS = - #{data.stepStatus, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - PROJECT_STATUS = #{data.projectStatus}, - STATUS_DESC = #{data.statusDesc}, - FINISH_TIME = #{data.finishTime} - WHERE PROJECT_ID = #{data.projectId} - - - - - - UPDATE CLOUD_LAB_TEST_TASK - SET - STEP_STATUS = - #{data.stepStatus, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - PROJECT_STATUS = #{data.projectStatus}, - STATUS_DESC = #{data.statusDesc}, - FINISH_TIME = #{data.finishTime}, - RESULT_FILE = - #{data.resultFile, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - TEST_SUMMARY = - #{data.testSummary, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - TEST_DETAIL = - #{data.testDetail, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - SCRIPT_RESULT_CONFIG = - #{data.scriptResultConfig, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler} - WHERE PROJECT_ID = #{data.projectId} - - - - - UPDATE CLOUD_LAB_TEST_TASK - SET - STEP_STATUS = - #{data.stepStatus, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - PROJECT_STATUS = #{data.projectStatus}, - STATUS_DESC = #{data.statusDesc}, - FINISH_TIME = #{data.finishTime}, - TEST_SUMMARY = - #{data.testSummary, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler}, - TEST_DETAIL = - #{data.testDetail, typeHandler=com.huawei.ic.openlab.cloudtest.common.typehandler.JacksonTypeHandler} - WHERE PROJECT_ID = #{data.projectId} - - - \ No newline at end of file diff --git a/component/CloudTest/src/main/resources/mapper/PerformanceServiceMapper.xml b/component/CloudTest/src/main/resources/mapper/PerformanceServiceMapper.xml deleted file mode 100644 index 6ad8b70c4ee9719c14f27e33ed7c6a22848cb5c1..0000000000000000000000000000000000000000 --- a/component/CloudTest/src/main/resources/mapper/PerformanceServiceMapper.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - UPDATE PERFORMANCE_SERVICE - SET TASK_NUM = TASK_NUM +1 - WHERE SERVICE_IP = #{serviceIp} - - - - UPDATE PERFORMANCE_SERVICE - SET TASK_NUM = TASK_NUM -1 - WHERE SERVICE_IP = #{serviceIp} - - \ No newline at end of file diff --git a/component/CompatibilityTesting/check_install_result.sh b/component/CompatibilityTesting/check_install_result.sh index c854e0b8883b64c58176ef8c4187798c0ce298d8..b35d7f9e1e56b4c2477abf389cff0cf740d4d4f2 100644 --- a/component/CompatibilityTesting/check_install_result.sh +++ b/component/CompatibilityTesting/check_install_result.sh @@ -1,3 +1,12 @@ #!/bin/bash -echo "true" \ No newline at end of file +compatibility_test_path=$(which compatibility_test) +if [[ ${compatibility_test_path} == ${HOME}/.local/compatibility_testing/bin/compatibility_test ]]; then + echo "true" +else + if [[ -f ${HOME}/.local/compatibility_testing/bin/compatibility_test ]] && [[ $(grep '^export PATH=${HOME}/.local/compatibility_testing/bin:${PATH}$' ${HOME}/.bashrc | wc -l) == "1" ]]; then + echo "true" + else + echo "false" + fi +fi \ No newline at end of file diff --git a/component/CompatibilityTesting/install.sh b/component/CompatibilityTesting/install.sh index 4bfa698639238723c071bc37114a5e28080e39a9..96ed4c585aad3b97073b1d3fa96e22000a80330d 100644 --- a/component/CompatibilityTesting/install.sh +++ b/component/CompatibilityTesting/install.sh @@ -4,10 +4,18 @@ set -e function main (){ - compatibility_test_tar=/tmp/devkitdependencies/compatibility_testing.tar.gz - echo "Decompress compatibility_testing.tar.gz to ${HOME}/.local" - tar --no-same-owner -zxf ${compatibility_test_tar} -C ${HOME}/.local/ - echo "Decompress compatibility_testing.tar.gz to ${HOME}/.local finished." + compatibility_test_tar=/tmp/devkitdependencies/compatibility_testing.tar.gz + echo "Decompress compatibility_testing.tar.gz to ${HOME}/.local" + tar --no-same-owner -zxf ${compatibility_test_tar} -C ${HOME}/.local/ + echo "Decompress compatibility_testing.tar.gz to ${HOME}/.local finished." + + echo "change ${HOME}/.bashrc" + sed -i '/compatibility_testing/d' ${HOME}/.bashrc + echo 'export PATH=${HOME}/.local/compatibility_testing/bin:${PATH}' >> ${HOME}/.bashrc + + echo "source ${HOME}/.bashrc" + set +x + source ${HOME}/.bashrc } main "$@" \ No newline at end of file diff --git a/component/DevKitCLI/check_install_result.sh b/component/DevKitCLI/check_install_result.sh index adf662f941de585666426a484618fc916d442101..81bc7810e6b5db9805a317399baf503cf888b2d1 100644 --- a/component/DevKitCLI/check_install_result.sh +++ b/component/DevKitCLI/check_install_result.sh @@ -1,6 +1,7 @@ #!/bin/bash -if [[ -f ${HOME}/.local/DevKit-CLI-24.0.RC1-Linux-Kunpeng/devkit ]]; then +devkit_path=$(which devkit) +if [[ -f ${HOME}/.local/DevKit-CLI-24.0.RC1-Linux-Kunpeng/devkit ]] && [[ ${devkit_path} == /usr/local/bin/devkit ]]; then echo "true" else echo "false" diff --git a/component/DevKitCLI/install.sh b/component/DevKitCLI/install.sh index 78706aeb727ec186cde02375795e1c8547049711..ecf4a0a8e93226da127360c3209fa02549a754b1 100644 --- a/component/DevKitCLI/install.sh +++ b/component/DevKitCLI/install.sh @@ -7,4 +7,4 @@ echo "Decompress DevKit-CLI-24.0.RC1-Linux-Kunpeng.tar.gz to ${HOME}/.local" tar --no-same-owner -zxf /tmp/devkitdependencies/DevKit-CLI-24.0.RC1-Linux-Kunpeng.tar.gz -C ${HOME}/.local echo "Decompress DevKit-CLI-24.0.RC1-Linux-Kunpeng.tar.gz to ${HOME}/.local finished." sudo ln -s ${HOME}/.local/DevKit-CLI-24.0.RC1-Linux-Kunpeng/devkit /usr/local/bin -echo "install DevkitCLI success." +echo "create DevkitCLI soft-link success." diff --git a/component/DevKitTester/build_devkit_tester.sh b/component/DevKitTester/build_devkit_tester.sh index 0eb1f419e5f060f1bb8a4c685b0b4ba654e4f2d7..f89d71b804734570b856b7010a015e0c3350cbc3 100644 --- a/component/DevKitTester/build_devkit_tester.sh +++ b/component/DevKitTester/build_devkit_tester.sh @@ -46,6 +46,6 @@ cp -rf "${current_dir}"/JFRParser/target/JFRParser/* devkit_tester tar -czf devkit_tester.tar.gz devkit_tester -mkdir -p "${project_dir}"/build/component/DevKitTester -cp "${current_dir}/check_install_result.sh" "${project_dir}"/build/component/DevKitTester -cp "${current_dir}/install.sh" "${project_dir}"/build/component/DevKitTester +# mkdir -p "${project_dir}"/build/component/DevKitTester +# cp -rf "${current_dir}/check_install_result.sh" "${project_dir}"/build/component/DevKitTester +# cp -rf "${current_dir}/install.sh" "${project_dir}"/build/component/DevKitTester diff --git a/component/DevKitWeb/devkit_installer/devkit_installer.py b/component/DevKitWeb/devkit_installer/devkit_installer.py index a89ee3a2aa3b266fa6087b1a574f2ae05801503d..71d396316f5942dbc0b3ee5d6fd51ab5f2706b0d 100644 --- a/component/DevKitWeb/devkit_installer/devkit_installer.py +++ b/component/DevKitWeb/devkit_installer/devkit_installer.py @@ -41,10 +41,7 @@ class DevKitInstaller: plugin="java_perf", rpc_port=50051) if not install_result: - LOGGER.error(f"Install DevKit failed. Here are some reasons \n" - f"1. /opt directory permission is not 757, plz change it to 757\n" - f"2. the install user doesn't have sudo permission\n" - f"3. the install user doesn't have sudo password-free\n") + LOGGER.error(f"Install DevKit failed.") sys.exit(1) LOGGER.info(f"Install DevKit success.") diff --git a/component/DevKitWeb/devkit_installer/devkit_machine.py b/component/DevKitWeb/devkit_installer/devkit_machine.py index fb477a0958529a04448241e425029e3380bba128..5e38485b23088b593451441e3abf2284d50cda69 100644 --- a/component/DevKitWeb/devkit_installer/devkit_machine.py +++ b/component/DevKitWeb/devkit_installer/devkit_machine.py @@ -70,10 +70,15 @@ class DevKitMachine: ssh_client = self.ssh_client() if not ssh_client: return False - ret = self.decompress_package_handle(ssh_client, package_path, package_name) + try: + ret = self.decompress_package_handle(ssh_client, package_path, package_name) + except (timeout_decorator.TimeoutError, Exception) as e: + LOGGER.error(f"Remote machine {self.ip} install DevKitWeb occur Error: decompress package failed. {str(e)}") + ret = False ssh_client.close() return ret + @timeout_decorator.timeout(60) def decompress_package_handle(self, ssh_client, package_dir, package_name): package_file = os.path.join(package_dir, package_name) stdin, stdout, stderr = ssh_client.exec_command(f"ls {package_file}") @@ -131,14 +136,20 @@ class DevKitMachine: while result.find(error_special_end) != -1: # 授权工具安装继续运行 - result = self.channel_send_cmd(channel=channel, cmd="y\n", special_end=special_end, - error_special_end=error_special_end, timeout=300) + try: + result = self.channel_send_cmd(channel=channel, cmd="y\n", special_end=special_end, + error_special_end=error_special_end, timeout=300) + except (timeout_decorator.TimeoutError, Exception) as e: + LOGGER.error(f"Remote machine {self.ip} install DevKitWeb occur Error: {str(e)}. " + f"Please install DevKitWeb manually.") + if result.find(special_end) != -1: ssh_client.close() return True ssh_client.close() return False + @timeout_decorator.timeout(300) def channel_send_cmd(self, channel, cmd, special_end, error_special_end, timeout=60): buff_decode = "" try: diff --git a/component/GCCforOpenEuler/install.sh b/component/GCCforOpenEuler/install.sh index c9730531787eec84ac4bf1d8400d190f5cca6539..538cc19472c1d388444b44539b51f4cdf55869cc 100644 --- a/component/GCCforOpenEuler/install.sh +++ b/component/GCCforOpenEuler/install.sh @@ -1,35 +1,30 @@ #!/bin/bash cd /tmp/devkitdependencies/ -verify_signature=$(sha256sum -c gcc-10.3.1-2023.12-aarch64-linux.tar.gz.sha256 >/dev/null 2>&1; echo $?) -if [[ ${verify_signature} -eq "0" ]]; then - if [[ ! -d ${HOME}/.local/gcc-10.3.1-2023.12-aarch64-linux ]]; then - mkdir -p ${HOME}/.local - echo "Decompress gcc-10.3.1-2023.12-aarch64-linux.tar.gz to ${HOME}/.local" - tar --no-same-owner -zxf /tmp/devkitdependencies/gcc-10.3.1-2023.12-aarch64-linux.tar.gz -C ${HOME}/.local - echo "Decompress gcc-10.3.1-2023.12-aarch64-linux.tar.gz to ${HOME}/.local finished." - fi +if [[ ! -d ${HOME}/.local/gcc-10.3.1-2023.12-aarch64-linux ]]; then + mkdir -p ${HOME}/.local - gcc_path=$(which gcc) - if [[ ${gcc_path} != ${HOME}/.local/gcc-10.3.1-2023.12-aarch64-linux/bin/gcc ]]; then - sed -i '/.*export GCC_HOME=${HOME}\/.local/d' ${HOME}/.bashrc - sed -i '/.*export .*=${GCC_HOME}.*/d' ${HOME}/.bashrc + echo "Decompress gcc-10.3.1-2023.12-aarch64-linux.tar.gz to ${HOME}/.local" + tar --no-same-owner -zxf /tmp/devkitdependencies/gcc-10.3.1-2023.12-aarch64-linux.tar.gz -C ${HOME}/.local + echo "Decompress gcc-10.3.1-2023.12-aarch64-linux.tar.gz to ${HOME}/.local finished." +fi - echo "change ${HOME}/.bashrc" - cat >> ${HOME}/.bashrc <<'EOF' +gcc_path=$(which gcc) +if [[ ${gcc_path} != ${HOME}/.local/gcc-10.3.1-2023.12-aarch64-linux/bin/gcc ]]; then + sed -i '/.*export GCC_HOME=${HOME}\/.local/d' ${HOME}/.bashrc + sed -i '/.*export .*=${GCC_HOME}.*/d' ${HOME}/.bashrc + + echo "change ${HOME}/.bashrc" + cat >> ${HOME}/.bashrc <<'EOF' export GCC_HOME=${HOME}/.local/gcc-10.3.1-2023.12-aarch64-linux export PATH=${GCC_HOME}/bin:${PATH} export INCLUDE=${GCC_HOME}/include:${INCLUDE} export LD_LIBRARY_PATH=${GCC_HOME}/lib64:${LD_LIBRARY_PATH} EOF - echo "source ${HOME}/.bashrc" - set +x - source ${HOME}/.bashrc - else - echo "install gcc-10.3.1-2023.12 success." - fi - + echo "source ${HOME}/.bashrc" + set +x + source ${HOME}/.bashrc else - echo "Failed to verify the signature of the gcc-10.3.1-2023.12-aarch64-linux.tar.gz installation package." -fi \ No newline at end of file + echo "install gcc-10.3.1-2023.12 success." +fi diff --git a/component/LkpTests/build_lkp_tests_all.sh b/component/LkpTests/build_lkp_tests_all.sh deleted file mode 100644 index 4756c68ca315f2e95bce2940156e7e93fbc8e140..0000000000000000000000000000000000000000 --- a/component/LkpTests/build_lkp_tests_all.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# SourceCode build script -# Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. - -current_dir=$(cd $(dirname $0); pwd) - -function main() { - upload_path=$1 - echo "build LkpTests upload_path: " "${upload_path}" - sh ${current_dir}/lkp_help/lkp_help.sh ${upload_path} - if [[ "$?" -ne "0" ]]; then - exit 1 - fi - - cd ${current_dir}/compatibility_help/ - wget -c https://gitee.com/jerry-553/lkp_test_devkitpipeline/releases/download/lkp-all-resource/compatibility_testing.tar.gz - if [[ "$?" -ne "0" ]]; then - exit 1 - fi - sh ${current_dir}/compatibility_help/compatibility_help.sh ${current_dir}/compatibility_help ${upload_path} - if [[ "$?" -ne "0" ]]; then - exit 1 - fi - - /bin/cp -rf ${current_dir}/gem_dependencies.zip ${upload_path} - if [[ "$?" -ne "0" ]]; then - exit 1 - fi -} - -main "$@" diff --git a/component/LkpTests/check_install_result.sh b/component/LkpTests/check_install_result.sh deleted file mode 100644 index 226f9eea1507ad74b771deb3acf19e3d8fac2987..0000000000000000000000000000000000000000 --- a/component/LkpTests/check_install_result.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -lkp_path=$(which lkp) -if [[ "${lkp_path}" == *"/local/bin/lkp"* ]]; then - echo "true" -else - echo "false, 请确认是否配everything的yum源，以及是否给安装用户配属sudo权限" -fi \ No newline at end of file diff --git a/component/LkpTests/compatibility_help/check_report_result.py b/component/LkpTests/compatibility_help/check_report_result.py deleted file mode 100644 index b97cfe8e3549979beb1efd3d471132b352a4934c..0000000000000000000000000000000000000000 --- a/component/LkpTests/compatibility_help/check_report_result.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import json - -COMPATIBILITY_REPORT_ENTRIES = ["id", "result", "reason", "evidence"] -COMPATIBILITY_REPORT_TEMPLATE_HOLDER = "/** compatibility-test-report **/" -COMPATIBILITY_JSON_NAME = "log.json" - -flag = 0 -with open(os.path.join(os.getcwd(), "Chinese", COMPATIBILITY_JSON_NAME), "r") as file: - data = json.load(file) - for each in data: - for each_key in COMPATIBILITY_REPORT_ENTRIES: - if each_key == "result": - if each.get(each_key) and each.get(each_key) == "failed": - flag = -1 - break - -print(flag) \ No newline at end of file diff --git a/component/LkpTests/compatibility_help/compatibility_help.sh b/component/LkpTests/compatibility_help/compatibility_help.sh deleted file mode 100644 index 6c8dfdcde1652ad4951cb8cc3a221b9e771f543e..0000000000000000000000000000000000000000 --- a/component/LkpTests/compatibility_help/compatibility_help.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# SourceCode build script -# Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. - -set -e -function main() { - save_path=$1 - upload_path=$2 - cd ${save_path} - - echo "/bin/cp -rf compatibility_testing.tar.gz ${upload_path}" - /bin/cp -rf compatibility_testing.tar.gz ${upload_path} -} - -main "$@" diff --git a/component/LkpTests/compatibility_help/compatibility_testing.sh b/component/LkpTests/compatibility_help/compatibility_testing.sh deleted file mode 100644 index df3d204b0a37990ab3424f8c46818ea11d353cdf..0000000000000000000000000000000000000000 --- a/component/LkpTests/compatibility_help/compatibility_testing.sh +++ /dev/null @@ -1,1215 +0,0 @@ -#!/bin/bash -################################## -#功能描述: 提供给用户进行兼容性测试、指标日志采集工具 -#版本信息: 华为技术有限公司，版权所有（C） 2020-2023 -#修改记录：2023-04-08 修改 -################################## -# 创建日志目录 -clear -if [[ ! -d "./log" ]]; then - mkdir ./log -fi -CURRENT_PATH=$(pwd) -APP_PID="" -STRESS_CMD=-1 -LIB_PATH=${CURRENT_PATH}/../lib/ -REMOTE_EXECUTIVE_PATH=/home/compatibility_testing/Chinese/ -REMOTE_LIB_PATH=/home/compatibility_testing/lib/ -current_time=$(date "+%Y%m%d") -log_file=info.log_${current_time} -error_file=error.log_${current_time} -app_log_file=app_log.log_${current_time} -nmap_log_file=nmap.log_${current_time} -# 清空日志文件。 -log_files=( "${CURRENT_PATH}"/log/"${error_file}" "${CURRENT_PATH}"/log/"${app_log_file}" "${CURRENT_PATH}"/log/"${nmap_log_file}") -for file in "${log_files[@]}"; do - if [[ -e "${file}" ]]; then - cat /dev/null >"${file}" - else - touch ${file} - fi -done - -# 是否有配置测试工具启动命令。 -TEST_TOOL_COMM=0 -# 是否有配置应用启动命令。 -START_APP_COMM=0 -# 是否有配置应用停止命令。 -STOP_APP_COMM=0 -# 命令执行失败后，手动停止业务应用。 -HAND_STOP_APP=0 -# 命令执行失败后，手动启动业务应用。 -HAND_START_APP=0 -# 命令执行失败后，手动启动测试工具。 -HAND_START_TEST=0 -# 判断是否安装鲲鹏开发套件 -HAS_KUNPENG_DEVKIT=0 -SYS_LOG_="messages" -DEBUG=0 -# 判断是否集群部署 -HAS_CLUSTER_ENV=0 -KUBERNETES_ENV=0 -CLAMAV_SCAN=0 -CVE_SCAN=0 -HPC_CERTIFICATE=0 -BINARY_PACK=0 -CURRENT_SYS="" -O_S_VERSION="" -declare -a product_result_array -declare -a snapshot_result_array -declare -a performance_test_pid_array - -write_messages() { - # 日志输出函数 - # 参数1：输出日志级别 - # 参数2：输出颜色，0-默认，31-红色，32-绿色，33-黄色，34-蓝色，35-紫色，36-天蓝色，3-白色。 - # 参数3：执行步骤。 - # 参数4：输出的日志内容。 - DATE=$(date "+%Y-%m-%d %H:%M:%S") - messages=$4 - step=$3 - level_info=$1 - colors=$2 - case ${level_info} in - i) echo "#${DATE}#info#${step}#${messages}" >> ./log/"${log_file}" - ;; - e) echo "#${DATE}#error#${step}#${messages}" >> ./log/"${log_file}" - echo -e "\033[1;31m${messages}\033[0m" - ;; - m) echo "#${DATE}#value#${step}#${messages}" >> ./log/"${log_file}" ;; - s) echo "#${DATE}#serious#${step}#${messages}" >> ./log/"${error_file}" - echo -e "\033[1;31m${messages}\033[0m" - ;; - c) echo -e "\033[1;${colors}m${messages}\033[0m" - echo "#${DATE}#info#${step}#${messages}" >> ./log/"${log_file}" - ;; - esac -} - -notice_users() { - write_messages c 34 0 "自动化兼容性测试开始前，请用户先填写配置文件compatibility_testing.conf，填写说明请参考README" - write_messages c 34 0 "自动化兼容性测试开始执行，脚本分为10个步骤，运行时间约15分钟，请耐心等待。" - write_messages i 0 0 "自动化测试采集工具开始执行。" -} - -production_env_waring() { - write_messages c 31 0 "测试期间会不断启动和停止待测试应用软件，请勿在生产环境执行兼容性测试工具。确认当前环境不是生产环境，请回复 N ，是生产环境请回复 Y ?" - read -r INPUT_ - write_messages c 31 0 "您输入的是 ${INPUT_} " - if [[ "${INPUT_}" == "Y" || "${INPUT_}" == "y" ]]; then - exit - fi -} - - -check_configuration() { - # 检查用户填写配置项是否为空 - config_file="${CURRENT_PATH}""/compatibility_testing.conf" - - if [[ -f "${config_file}" ]]; then - application_names=$(sed '/^application_names=/!d;s/application_names=//' "${config_file}") - start_app_commands=$(sed '/^start_app_commands=/!d;s/start_app_commands=//' "${config_file}") - stop_app_commands=$(sed '/^stop_app_commands=/!d;s/stop_app_commands=//' "${config_file}") - start_performance_scripts=$(sed '/^start_performance_scripts=/!d;s/start_performance_scripts=//' "${config_file}") - cluster_ip_lists=$(sed '/^cluster_ip_lists=/!d;s/cluster_ip_lists=//' "${config_file}") - kubernetes_env=$(sed '/^kubernetes_env=/!d;s/kubernetes_env=//' "${config_file}") - cve_scan_path=$(sed '/^cve_scan_path=/!d;s/cve_scan_path=//' "${config_file}") - clamav_scan_path=$(sed '/^clamav_scan_path=/!d;s/clamav_scan_path=//' "${config_file}") - hpc_certificate=$(sed '/^hpc_certificate=/!d;s/hpc_certificate=//' "${config_file}") - binary_file=$(sed '/^binary_file=/!d;s/binary_file=//' "${config_file}") - if [[ -z "${application_names}" ]]; then - write_messages e 31 2 "配置文件中的应用名称为空,请填写正确后重启脚本。" - exit - fi - if [[ -z "${start_app_commands}" ]]; then - write_messages e 31 2 "应用启动命令为空。" - START_APP_COMM=1 - fi - if [[ -z "${stop_app_commands}" ]]; then - write_messages e 31 2 "应用停止命令为空。" - STOP_APP_COMM=1 - fi - export STRESS_CMD - if [[ -z "${start_performance_scripts}" ]]; then - write_messages e 31 2 "压力测试工具启动命令为空。" - - STRESS_CMD=0 - TEST_TOOL_COMM=1 - else - STRESS_CMD=1 - fi - if [[ -n "${cluster_ip_lists}" ]]; then - HAS_CLUSTER_ENV=1 - fi - if [[ -n "${kubernetes_env}" && "${kubernetes_env}" == "Y" ]];then - KUBERNETES_ENV=1 - fi - if [[ -n "${cve_scan_path}" ]];then - CVE_SCAN=1 - fi - if [[ -n "${clamav_scan_path}" ]];then - CLAMAV_SCAN=1 - fi - if [[ -n "${hpc_certificate}" ]];then - HPC_CERTIFICATE=1 - fi - if [[ -n "${binary_file}" ]];then - BINARY_PACK=1 - fi - else - write_messages e 31 2 "配置文件不存在，请检查。" - exit - fi -} - -create_result_dir() { - # 创建采集目录 - write_messages i 0 1 "创建目录" - result_dirs=("data/hardware/" "data/software/" "data/system/" "data/product/" - "data/test/performance/" "data/test/compatiable/" "data/test/function/" - "data/test/power/" "data/test/safety" "data/test/dfx" "data/others") - for dir in "${result_dirs[@]}"; do - if [[ ! -d "${dir}" ]]; then - mkdir -p "${dir}" - else - find ./"${dir}" -type f -name "*.log*" -exec rm {} \; - fi - done - # 删除远端服务器的文件 - OLD_IFS="${IFS}" - IFS=',' - read -r -a ip_list <<< "$cluster_ip_lists" - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - ssh root@${ip_addr} "if [ -d ${REMOTE_EXECUTIVE_PATH} ];then cd ${REMOTE_EXECUTIVE_PATH}; rm -rf data log;fi" - done - fi - IFS="${OLD_IFS}" -} - -get_service_info() { - # 获取服务器信息 - OLD_IFS="${IFS}" - IFS=',' - read -r -a ip_list <<< "$cluster_ip_lists" - path="data/hardware/" - if [[ ! -d "${path}" ]]; then - mkdir -p "${path}" - fi - path="data/software/" - if [[ ! -d "${path}" ]]; then - mkdir -p "${path}" - fi - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - scp -r ${CURRENT_PATH}/obtain_service_info.sh ${ip_addr}:${REMOTE_EXECUTIVE_PATH} >/dev/null - scp ${CURRENT_PATH}/../lib/cvecheck ${ip_addr}:${REMOTE_LIB_PATH} >/dev/null - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; chmod +x obtain_service_info.sh; bash obtain_service_info.sh" - done - bash obtain_service_info.sh - else - bash obtain_service_info.sh - fi - - if [[ ${BINARY_PACK} -eq 1 ]]; then - if [[ -f ${binary_file} ]]; then - - gcc_file="data/software/system_version.log" - echo -e "\n=${binary_file}=" >>${gcc_file} 2>> /dev/null; - echo -e "\n=============binary files=============" >>${gcc_file} 2>> /dev/null; - strings ${binary_file} |grep -E 'GCC|gcc|clang' >>${gcc_file} 2>> /dev/null; - else - write_messages e 0 4 "配置文件填写的二进制文件不存在。" - fi - fi - - file_name_list=("data/hardware/hardware_info.log" "data/hardware/hardware_pcie.log" "data/hardware/hardware_cpu.log" - "data/hardware/hardware_disk.log" "data/software/system_version.log") - smartctl_file_name="data/hardware/hardware_smartctl.log" - command_desc=("服务器型号" "pci信息" "CPU信息" "硬盘分区" "内核信息") - length=${#file_name_list[@]} - for ((i = 0; i < "${length}"; i++)); do - file_name=${file_name_list[$i]} - if [[ -f ${file_name} ]]; then - product_result_array[$i]='True' - else - product_result_array[$i]='False' - fi - done - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - ip_length=${#ip_list[@]} - for ((j =0;j<"${ip_length}";j++));do - ip_addr=${ip_list[$j]} - for ((i = 0; i < "${length}"; i++)); do - file_name=${file_name_list[$i]} - index=$((length * (j + 1) + i)) - scp ${ip_addr}:${REMOTE_EXECUTIVE_PATH}${file_name} ${file_name}_${ip_addr} >/dev/null - if [[ $? -eq 0 ]];then - product_result_array[$index]='True' - else - product_result_array[$index]='False' - fi - done - - - scp ${ip_addr}:${REMOTE_EXECUTIVE_PATH}${smartctl_file_name} ${smartctl_file_name}_${ip_addr} >/dev/null - - done - fi - IFS="${OLD_IFS}" -} - -get_ps_snapshot() { - # 获取服务器进程的快照 - OLD_IFS="${IFS}" - IFS=',' - read -r -a ip_list <<< "$cluster_ip_lists" - file_path="data/product/" - if [[ ! -d "${file_path}" ]]; then - mkdir -p "${file_path}" - fi - file_name='product_name.log' - index=0 - if ! ps aux >${file_path}${file_name}; then - snapshot_result_array[$index]='False' - write_messages e 0 5 "调用ps aux命令获取服务器进程快照失败" - else - snapshot_result_array[$index]='True' - fi - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - index=$(( index + 1 )) - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; mkdir -p ${file_path}; ps aux >${file_path}${file_name}" - scp ${ip_addr}:${REMOTE_EXECUTIVE_PATH}${file_path}${file_name} ${file_path}${file_name}_${ip_addr} >/dev/null - if [[ $? -eq 0 ]];then - snapshot_result_array[$index]='True' - else - snapshot_result_array[$index]='False' - fi - done - fi - IFS="${OLD_IFS}" -} - -check_system_message() { - # 检查系统日志 - # 参数1：开始时间 - # 参数2：结束时间 - start_time=$1 - end_time=$2 - file_path="data/system/" - file_name="message.log" - error_file_name="err_messages.log" - if [[ ! -d "${file_path}" ]]; then - mkdir -p "${file_path}" - fi - system_message_result='True' - if [[ ${start_time}x == ""x || ${end_time}x == ""x ]]; then - grep -i -E 'fail|error' /var/log/"${SYS_LOG_}" > "${file_path}""${error_file_name}" - else - if ! sed -n /"${start_time}"/,/"${end_time}"/p /var/log/"${SYS_LOG_}" >"${file_path}""${file_name}"; then - system_message_result='False' - write_messages e 0 9 "执行获取服务器日志失败" - write_messages s 0 9 "调用sed -n '/${start_time}/,/${end_time}/p' /var/log/${SYS_LOG_}命令失败" - else - grep -i -E 'fail|error' "${file_path}""${file_name}" > "${file_path}""${error_file_name}" - fi - fi -} - -tar_output() { - OLD_IFS="${IFS}" - IFS=',' - read -r -a ip_list <<< "$cluster_ip_lists" - current_time=$(date "+%Y%m%d%H%M%S") - # 检查文件是否存在 - result_files=("data/hardware/hardware_cpu.log" "data/hardware/hardware_disk.log" - "data/hardware/hardware_info.log" "data/hardware/hardware_pcie.log" - "data/product/product_name.log" "data/software/system_version.log" - "data/test/performance/test_perf_cpu_1.log" "data/test/performance/test_perf_disk_1.log" - "data/test/performance/test_perf_mem_1.log" - "data/test/performance/test_perf_net_1.log" "data/test/compatiable/test_perf_cpu_0.log" - "data/test/compatiable/test_perf_cpu_1.log" "data/test/compatiable/test_perf_disk_0.log" - "data/test/compatiable/test_perf_disk_1.log" "data/test/compatiable/test_perf_mem_0.log" - "data/test/compatiable/test_perf_mem_1.log" "data/test/compatiable/test_perf_net_0.log" - "data/test/compatiable/test_perf_net_1.log") - for data_file in "${result_files[@]}"; do - if [[ ! -f "${data_file}" ]]; then - write_messages e 0 10 "数据目录下的日志文件${data_file}不存在" - fi - done - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - for data_file in "${result_files[@]}"; do - data_file=${data_file}_${ip_addr} - if [[ ! -f "${data_file}" ]]; then - write_messages e 0 10 "数据目录下的日志文件${data_file}不存在" - fi - done - done - fi - - record_log_file=( "${error_file}" "${app_log_file}" "${log_file}") - for file in "${record_log_file[@]}"; do - if [[ -f ./log/"${file}" ]]; then - cp ./log/"${file}" ./data/others/ - else - write_messages e 0 10 "./log/目录下的日志文件${file}不存在" - fi - done - if [[ -f "${config_file}" ]]; then - cp "${config_file}" ./data/ - fi - if ! tar -czf log.tar.gz data; then - write_messages e 0 10 "压缩文件出错" - fi - write_messages c 34 10 "采集结束，日志打包完成，压缩包log.tar.gz存放在$CURRENT_PATH。" - IFS=${OLD_IFS} -} - -check_error() { - # 检查采集期间是否有异常 - OLD_IFS="${IFS}" - IFS=',' - read -r -a ip_list <<< "$cluster_ip_lists" - for ((i = 0; i < ${#product_result_array[@]}; i++)); do - if [[ "${product_result_array[i]}" = 'False' ]]; then - index=$(($i/5)) - sub_index=$(($i%5)) - if [[ ${index} -eq 0 ]]; then - desc="本服务器" - else - desc=${ip_list[$(($index-1))]} - fi - case ${sub_index} in - 0) write_messages e 0 10 "在${desc}执行获取服务器型号命令失败" ;; - 1) write_messages e 0 10 "在${desc}执行pcie命令失败" ;; - 2) write_messages e 0 10 "在${desc}执行lscpu命令失败" ;; - 3) write_messages e 0 10 "在${desc}执行lsblk命令失败" ;; - 4) write_messages e 0 10 "在${desc}获取操作系统内核版本失败" ;; - esac - fi - done - if [[ "${system_message_result}" = 'False' ]]; then - write_messages e 0 10 "执行检查系统日志失败，请检查/var/log/${SYS_LOG_}是否有权限查询" - fi - for ((i = 0; i < ${#snapshot_result_array[@]}; i++)); do - if [[ "${snapshot_result_array[i]}" = 'False' ]];then - if [[ ${i} -eq 0 ]]; then - write_messages e 0 10 "在本服务器执行ps查看进程失败" - else - index=$(( $i -1)) - write_messages e 0 10 "在${ip_list[index]}执行ps查看进程失败" - fi - - fi - done - - if [[ -f ./log/"${error_file}" ]]; then - while read -r line; do - echo "${line}" | awk -F'#' '{if($5~/sar/) print $5}' - done <./log/"${error_file}" - fi - IFS="${OLD_IFS}" -} - -# 集群环境检查各IP的是否连通，并获取操作系统版本。 -check_connection_and_OS() { - OLD_IFS="${IFS}" - IFS=',' - read -r -a ip_list <<< "$cluster_ip_lists" - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - ssh root@"${ip_addr}" "mkdir -p ${REMOTE_EXECUTIVE_PATH};mkdir -p ${REMOTE_LIB_PATH};" - if [[ $? -ne 0 ]]; then - write_messages e 0 1 "${ip_addr} 与当前服务器没有建立免密互信，请配置后重新执行脚本" - exit - fi - scp -r ${CURRENT_PATH}/env_OSVersion.sh ${ip_addr}:${REMOTE_EXECUTIVE_PATH} >/dev/null - os_version_=$(ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; chmod +x env_OSVersion.sh; bash env_OSVersion.sh") - if [[ ${O_S_VERSION}x == ""x ]];then - O_S_VERSION=${os_version_} - else - O_S_VERSION="${O_S_VERSION}"";""${os_version_}" - fi - done - - fi - os_version_=$(chmod +x *.sh; bash env_OSVersion.sh) - if [[ ${O_S_VERSION}x == ""x ]];then - O_S_VERSION=${os_version_} - else - O_S_VERSION="${O_S_VERSION}"";""${os_version_}" - fi - write_messages i 0 1 "当前的操作系统版本是${O_S_VERSION}。" - IFS="${OLD_IFS}" -} - -# 检查系统环境，安装依赖软件 -env_preparation() { - check_connection_and_OS - OLD_IFS="${IFS}" - IFS=',' - read -r -a ip_list <<< "$cluster_ip_lists" - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - ssh root@"${ip_addr}" "mkdir -p ${REMOTE_EXECUTIVE_PATH}" - scp -r ${CURRENT_PATH}/env_preparation.sh ${ip_addr}:${REMOTE_EXECUTIVE_PATH} > /dev/null - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; chmod +x env_preparation.sh; bash env_preparation.sh " - if [[ $? -ne 0 ]]; then - write_messages e 0 1 "${ip_addr} 安装依赖软件失败，失败原因请登录该服务器，查看目录${REMOTE_EXECUTIVE_PATH}/log/info.log 的日志" - exit - fi - done - bash env_preparation.sh - if [[ $? -ne 0 ]]; then - write_messages e 0 1 "安装依赖软件失败，失败原因请查看目录${REMOTE_EXECUTIVE_PATH}/log/info.log 的日志" - exit - fi - - else - bash env_preparation.sh - if [[ $? -ne 0 ]]; then - write_messages e 0 1 "安装依赖软件失败，失败原因请查看目录${REMOTE_EXECUTIVE_PATH}/log/info.log 的日志" - exit - fi - - fi - IFS="${OLD_IFS}" -} - -HCS8_PREFIX="172.36.0.10:58089/rest/v2/virtualMachine/verifyIsHCS?mac=" -HCS803_PREFIX="173.64.11.52:58088/rest/v2/virtualMachine/verifyIsHCS?mac=" -check_physical_system(){ - # 判断当前系统是否是物理服务器 - OLD_IFS="${IFS}" - IFS=',' - CURRENT_SYS="" - read -r -a ip_list <<< "$cluster_ip_lists" - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - CURRENT_SYS_=$(ssh root@"${ip_addr}" "dmidecode -s system-product-name") - if [[ ${CURRENT_SYS}x == ""x ]];then - CURRENT_SYS="${CURRENT_SYS_}" - else - CURRENT_SYS="${CURRENT_SYS}"";""${CURRENT_SYS_}" - fi - done - fi - IFS="${OLD_IFS}" - CURRENT_SYS_=$(dmidecode -s system-product-name) - if [[ ${CURRENT_SYS}x == ""x ]];then - CURRENT_SYS="${CURRENT_SYS_}" - else - CURRENT_SYS="${CURRENT_SYS}"";""${CURRENT_SYS_}" - fi -} - -stop_or_start_kunpengdeveloper(){ - step=$1 - OLD_IFS="${IFS}" - IFS=',' - read -r -a ip_list <<< "$cluster_ip_lists" - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - ssh root@"${ip_addr}" "mkdir -p ${REMOTE_EXECUTIVE_PATH}" - scp -r ${CURRENT_PATH}/env_kunpengdeveloper.sh ${ip_addr}:${REMOTE_EXECUTIVE_PATH} >/dev/null - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; chmod +x env_kunpengdeveloper.sh; bash env_kunpengdeveloper.sh ${step}" - done - bash env_kunpengdeveloper.sh ${step} - - else - bash env_kunpengdeveloper.sh ${step} - - fi - IFS="${OLD_IFS}" -} - -sys_env_inspectation() { - # 从CPU、内存、硬盘和网卡四个角度检查利用率是否过高，如果过高则提升当前环境非空闲 - write_messages i 0 3 "环境自检开始" - OLD_IFS="${IFS}" - IFS=',' - ENV_NOT_IDLE=0 - exam_time=0 - read -r -a ip_list <<< "$cluster_ip_lists" - while [[ "${exam_time}" -lt 5 ]]; do - ENV_NOT_IDLE=0 - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - ssh root@"${ip_addr}" "mkdir -p ${REMOTE_EXECUTIVE_PATH}" - scp -r ${CURRENT_PATH}/env_inspectation.sh ${ip_addr}:${REMOTE_EXECUTIVE_PATH} >/dev/null - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; chmod +x env_inspectation.sh; bash env_inspectation.sh" - if [[ $? -ne 0 ]]; then - write_messages e 0 1 "服务器为${ip_addr}的环境非空闲，请停止运行的业务软件，详细信息请登录该服务器，查看目录${REMOTE_EXECUTIVE_PATH}/log/info.log 的日志" - ENV_NOT_IDLE=1 - fi - done - bash env_inspectation.sh - if [[ $? -ne 0 ]]; then - write_messages e 0 1 "当前服务器的环境非空闲，请停止运行的业务软件，详细信息查看目录${REMOTE_EXECUTIVE_PATH}/log/info.log 的日志" - ENV_NOT_IDLE=1 - fi - else - bash env_inspectation.sh - if [[ $? -ne 0 ]]; then - write_messages e 0 1 "当前服务器的环境非空闲，请停止运行的业务软件，详细信息查看目录${REMOTE_EXECUTIVE_PATH}/log/info.log 的日志" - ENV_NOT_IDLE=1 - fi - fi - if [[ "${ENV_NOT_IDLE}" -eq 1 ]] ; then - exam_time=$((exam_time + 1)) - else - write_messages i 0 3 "环境自检结束" - break - fi - done - if [[ "${exam_time}" -eq 5 ]]; then - write_messages e 0 3 "环境自检没有通过，请用户确保停止所有业务应用及其依赖软件后重新执行脚本。如果需要跳过环境自检，请回复 Y。跳过环境自检会影响测试采集，导致测试结果不准确。如需停止脚本，请回复N。" - read -r INPUT_ - write_messages e 0 3 "您输入的是 ${INPUT_} " - if [[ "${INPUT_}" == "Y" || "${INPUT_}" == "y" ]]; then - write_messages i 0 3 "环境自检跳过" - else - exit - fi - fi - IFS="${OLD_IFS}" - -} - -port_scan() { - # 安全扫描，使用nmap进行端口扫描 - OLD_IFS="${IFS}" - IFS=',' - scan_pid="" - scan_process="" - read -r -a ip_list <<< "$cluster_ip_lists" - file_path="data/test/safety/" - if [[ ! -d "${file_path}" ]]; then - mkdir -p "${file_path}" - fi - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - { - scp ${CURRENT_PATH}/security_port_scan.sh ${ip_addr}:${REMOTE_EXECUTIVE_PATH} >/dev/null - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; chmod +x security_port_scan.sh; bash security_port_scan.sh" - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}${file_path}; for file in \$(find . -type f -name '*.nmap' -exec basename {} \; );do mv \${file} ${ip_addr}_\${file};done" - scp ${ip_addr}:${REMOTE_EXECUTIVE_PATH}${file_path}${ip_addr}*.nmap ${file_path} >/dev/null - } & - scan_pid=$! - if [[ -n ${scan_process} ]];then - scan_process="${scan_process},${scan_pid}" - else - scan_process=${scan_pid} - fi - done - for pid in ${scan_process};do - wait ${pid} - done - fi - bash security_port_scan.sh - write_messages i 0 6 "端口安全测试结束" - IFS="${OLD_IFS}" -} - -validated_clamav_scan(){ - if [[ "${CLAMAV_SCAN}" -eq 1 ]]; then - write_messages i 0 6 "防病毒扫描开始" - scan_path_parser 1 - write_messages i 0 6 "防病毒扫描结束" - fi -} - - -validated_cvecheck(){ - if [[ "${CVE_SCAN}" -eq 1 ]]; then - # 使用cvecheck进行漏洞扫描 - write_messages i 0 6 "漏洞扫描开始" - scan_path_parser 2 - write_messages i 0 6 "漏洞扫描结束" - fi -} - -scan_path_parser(){ - step=$1 - OLD_IFS="${IFS}" - IFS=',' - scan_pid="" - scan_process="" - scan_ip_desc=":" - current_server_path="" - file_path="data/test/safety/" - declare -A multiple_path_array - read -r -a ip_list <<< "$cluster_ip_lists" - if [[ "${step}" -eq 1 ]]; then - read -r -a scan_path_list <<< "$clamav_scan_path" - file_name=clam.log_${current_time} - else - read -r -a scan_path_list <<< "$cve_scan_path" - file_name=cvecheck-result.json - fi - if [[ ! -d "${file_path}" ]]; then - mkdir -p "${file_path}" - fi - for scan_path in "${scan_path_list[@]}"; do - if [[ ${scan_path} == *${scan_ip_desc}* ]]; then - ip=${scan_path%:*} - path=${scan_path#*:} - if [[ "${ip_list[@]}"=~"${ip}" ]]; then - if [[ -n ${multiple_path_array[$ip]} ]];then - multiple_path_array[$ip]="${multiple_path_array[$ip]},${path}" - else - multiple_path_array[$ip]=${path} - fi - fi - else - if [[ -n ${current_server_path} ]];then - current_server_path="${current_server_path},${scan_path}" - else - current_server_path=${scan_path} - fi - fi - done - - for ip_addr in "${ip_list[@]}"; do - if [[ -n ${multiple_path_array[$ip_addr]} ]]; then - if [[ "${step}" -eq 1 ]]; then - { - scp ${CURRENT_PATH}/security_clam_scan.sh ${ip_addr}:${REMOTE_EXECUTIVE_PATH} >/dev/null - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; chmod +x security_clam_scan.sh; bash security_clam_scan.sh ${multiple_path_array[$ip_addr]}" - scp ${ip_addr}:${REMOTE_EXECUTIVE_PATH}${file_path}${file_name} ${file_path}${file_name}_${ip_addr} >/dev/null - } & - else - { - scp ${CURRENT_PATH}/security_cvecheck.sh ${ip_addr}:${REMOTE_EXECUTIVE_PATH} >/dev/null - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; chmod +x security_cvecheck.sh; bash security_cvecheck.sh ${multiple_path_array[$ip_addr]}" - scp ${ip_addr}:${REMOTE_EXECUTIVE_PATH}${file_path}${file_name} ${file_path}${file_name}_${ip_addr} >/dev/null - } & - fi - scan_pid=$! - if [[ -n ${scan_process} ]];then - scan_process="${scan_process},${scan_pid}" - else - scan_process=${scan_pid} - fi - fi - done - for pid in ${scan_process};do - wait ${pid} - done - IFS="${OLD_IFS}" - if [[ -n ${current_server_path} ]];then - if [[ "${step}" -eq 1 ]]; then - bash security_clam_scan.sh ${current_server_path} - else - bash security_cvecheck.sh ${current_server_path} - fi - fi - unset scan_pid - unset scan_process - unset scan_ip_desc - unset current_server_path - unset multiple_path_array -} - - -check_process(){ - # 检查进程是否存在 - OLD_IFS="${IFS}" - IFS=',' - read -r -a ip_list <<< "$cluster_ip_lists" - process_name=$1 - process_name=$(echo ${process_name} | awk '{gsub(/^\s+|\s+$/, "");print}') - ps_results="" - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - if [[ ${KUBERNETES_ENV} -eq 1 ]];then - ps_result="$(ssh root@${ip_addr} "kubectl get all --all-namespaces |grep -i ${process_name} 2>/dev/null")" - else - ps_result="$(ssh root@${ip_addr} "pgrep -lf ${process_name}")" - fi - if [[ "${ps_result}" != '' ]]; then - ps_results="${ps_results}"";""${ps_result}" - fi - done - fi - if [[ ${KUBERNETES_ENV} -eq 1 ]];then - ps_result=$(kubectl get all --all-namespaces |grep -i ${process_name} 2>/dev/null) - else - ps_result=$(pgrep -lf "${process_name}") - fi - if [[ "${ps_result}" != '' ]]; then - ps_results="${ps_results}"";""${ps_result}" - fi - echo "${ps_results}" - IFS="${OLD_IFS}" -} - -check_process_exits_stop() { - # 检查业务应用是否存在，如果存在停止进程。 - step=$1 - OLD_IFS="${IFS}" - IFS=',' - read -r -a app_list <<< "$application_names" - read -r -a stop_comm_list <<< "$stop_app_commands" - length=${#app_list[@]} - sleep_time=10 - CURRENT_PATH=$(pwd) - STOP_APP_FLAG=0 - for ((i = 0; i < "${length}"; i++)); do - process=${app_list[$i]} - process=$(echo ${process} | awk '{gsub(/^\s+|\s+$/, "");print}') - ps_result="$(check_process ${process})" - if [[ "${ps_result}" != "" ]]; then - write_messages i 34 "${step}" "进程${process}存在" - STOP_APP_FLAG=1 - else - if [[ ${step} -ne 4 ]]; then - write_messages i 34 "${step}" "进程${process}不存在" - fi - fi - done - if [[ "${step}" -eq 9 && "${STOP_APP_FLAG}" -eq 0 ]] ; then - for stop_comm in "${stop_comm_list[@]}"; do - eval "${stop_comm}" >> "${CURRENT_PATH}"/log/"${app_log_file}" 2>&1 - if [[ "$?" -ne 0 ]]; then - cd "${CURRENT_PATH}"||exit - write_messages e 0 "${step}" "执行${stop_comm}出错" - else - cd "${CURRENT_PATH}"||exit - fi - done - fi - if [[ "${STOP_APP_COMM}" -eq 0 && "${STOP_APP_FLAG}" -eq 1 ]] ; then - for stop_comm in "${stop_comm_list[@]}"; do - eval "${stop_comm}" >> "${CURRENT_PATH}"/log/"${app_log_file}" 2>&1 - if [[ "$?" -ne 0 ]]; then - cd "${CURRENT_PATH}"||exit - write_messages e 0 "${step}" "执行${stop_comm}出错" - else - cd "${CURRENT_PATH}"||exit - fi - done - for ((i = 0; i < "${length}"; i++)); do - process=${app_list[$i]} - process=$(echo ${process} | awk '{gsub(/^\s+|\s+$/, "");print}') - check_times=5 - while [[ ${check_times} -gt 0 ]];do - sleep "${sleep_time}" - ps_result="$(check_process ${process})" - if [[ "${ps_result}" != "" ]]; then - write_messages e 0 "${step}" "执行${stop_comm}后，等待${sleep_time}秒，${process}存在" - kill_process ${process} - check_times=$((check_times -1)) - else - if [[ ${step} -ne 4 ]]; then - write_messages i 34 "${step}" "进程${process}不存在" - fi - break - fi - done - ps_result="$(check_process ${process})" - if [[ ${check_times} -le 0 ]] ; then - write_messages e 0 "${start_step}" "停止业务应用${process}失败，请用户检查停止脚本。" - HAND_STOP_APP=1 - fi - done - fi - if [[ "${STOP_APP_COMM}" -eq 1 ]] || [[ "${HAND_STOP_APP}" -eq 1 ]]; then - for ((i = 0; i < "${length}"; i++)); do - process=${app_list[$i]} - process=$(echo ${process} | awk '{gsub(/^\s+|\s+$/, "");print}') - ps_result="$(check_process ${process})" - check_times=5 - export APP_PID - echo $APP_PID - if ! [ -z "$APP_PID" ]; then - kill -9 $APP_PID - APP_PID="" - fi - if [[ ${check_times} -le 0 ]]; then - write_messages e 0 "${step}" "检查到应用程序还在启动，且用户未能停止应用，请用户停止应用后再执行脚本" - exit - else - write_messages i 0 "${step}" "进程${process}不存在" - fi - done - fi - IFS="${OLD_IFS}" -} - -kill_process() { - # 强制杀死进程 - OLD_IFS="${IFS}" - IFS=',' - process_name=$1 - kill_results="" - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - kill_result="$(ssh root@${ip_addr} "pgrep -f "${process}" | xargs kill -9")" - if [[ "${ps_result}" != '' ]]; then - kill_results="${kill_result}"";""${kill_result}" - fi - done - fi - kill_result="$(pgrep -f "${process}" | xargs kill -9 >>./log/"${log_file}" 2>&1)" - if [[ "${kill_result}" != '' ]]; then - kill_results="${kill_result}"";""${kill_result}" - fi - IFS="${OLD_IFS}" -} - -counting_time(){ - ds=$1 - ( - tput sc - for ((dsec = "${ds}"; dsec > 0; dsec--)); do - min=$((dsec / 60)) - se=$((dsec % 60)) - tput rc - tput ed - echo -ne "\r 采集剩余时间：${min}:${se}\r" - sleep 1 - done - ) & -} - - - -get_performance() { - # 进行CPU、内存、硬盘、网卡和功耗指标采集 - # 参数1：采集时间间隔 - # 参数2：采集时长 - # 参数3：采集序列，0：表示兼容测前采集，1：表示性能测试采集，2表示兼容测试后采集 - # 参数4：步骤 - OLD_IFS="${IFS}" - IFS=',' - frequency=$1 - seq=$3 - during_time=$2 - step=$4 - times=$((during_time * 60 / frequency)) - seconds=$((during_time * 60)) - if [[ "${seq}" -eq 1 ]]; then - file_path="data/test/performance/" - file_seq=1 - d_seconds=$((during_time * 60 *4)) - elif [[ "${seq}" -eq 0 ]]; then - file_path="data/test/compatiable/" - file_seq=0 - d_seconds=$((during_time * 60 *5)) - else - file_path="data/test/compatiable/" - file_seq=1 - d_seconds=$((during_time * 60 *5)) - fi - power_path="data/test/power/" - if [[ ! -d "${power_path}" ]]; then - mkdir -p "${power_path}" - fi - desc_array=('兼容性测试前采集' '性能测试采集' '兼容性测试后采集') - if [[ ! -d "${file_path}" ]]; then - mkdir -p "${file_path}" - fi - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - for ip_addr in "${ip_list[@]}"; do - { - ssh root@"${ip_addr}" "mkdir -p ${REMOTE_EXECUTIVE_PATH}; pgrep -f obtain_server_perf.sh | xargs kill -9 " - scp -r ${CURRENT_PATH}/obtain_server_perf.sh ${ip_addr}:${REMOTE_EXECUTIVE_PATH} >/dev/null - ssh root@"${ip_addr}" "cd ${REMOTE_EXECUTIVE_PATH}; chmod +x obtain_server_perf.sh; bash obtain_server_perf.sh ${frequency} ${during_time} ${seq} ${step}" - }& - done - fi - counting_time d_seconds - bash obtain_server_perf.sh ${frequency} ${during_time} ${seq} ${step} - file_list=("test_perf_cpu_${file_seq}.log" "test_perf_mem_${file_seq}.log" "test_perf_net_${file_seq}.log" "test_perf_disk_${file_seq}.log") - if [[ "${HAS_CLUSTER_ENV}" -eq 1 ]]; then - ip_length=${#ip_list[@]} - for ((j =0;j<"${ip_length}";j++));do - ip_addr=${ip_list[$j]} - for ((i = 0; i < 4; i++)); do - file_name=${file_list[$i]} - scp ${ip_addr}:${REMOTE_EXECUTIVE_PATH}${file_path}${file_name} ${file_path}${file_name}_${ip_addr} >/dev/null - done - if [[ "${seq}" -ne 1 ]]; then - file_name="test_power_${file_seq}.log" - scp ${ip_addr}:${REMOTE_EXECUTIVE_PATH}${power_path}${file_name} ${power_path}${file_name}_${ip_addr} >/dev/null - fi - done - fi - write_messages i 0 "${step}" "${desc_array[${seq}]}已完成" - IFS="${OLD_IFS}" -} - -start_app(){ - # 启动业务应用进程 - start_step=$1 - export APP_PID - if [[ ${start_step} -eq 5 ]];then - check_process_exits_stop "${start_step}" - fi - OLD_IFS="${IFS}" - IFS=',' - read -r -a app_list <<< "$application_names" - read -r -a start_comm_list <<< "$start_app_commands" - length=${#app_list[@]} - START_APP_FLAG=0 - SLEEP_TIME=10 - if [[ ${START_APP_COMM} -eq 0 ]];then - for start_comm in "${start_comm_list[@]}"; do - eval "${start_comm}" >> "${CURRENT_PATH}"/log/"${app_log_file}" 2>&1 - APP_PID=$! - if [ $? -ne 0 ]; then - cd "${CURRENT_PATH}"||exit - write_messages e 0 "${start_step}" "执行${start_comm}报错，请手动执行命令启动业务应用" - START_APP_FLAG=1 - fi - cd "${CURRENT_PATH}"||exit - done - for ((i = 0; i < "${length}"; i++)); do - process=${app_list[$i]} - process=$(echo ${process} | awk '{gsub(/^\s+|\s+$/, "");print}') - check_times=5 - while [[ ${check_times} -gt 0 ]];do - sleep "${SLEEP_TIME}" - ps_result="$(check_process ${process})" - if [[ "${ps_result}" != "" ]]; then - write_messages i 0 "${start_step}" "业务应用${process}启动完成。" - break - else - check_times=$((check_times -1)) - fi - done - if [[ ${check_times} -le 0 ]]; then - write_messages e 0 "${start_step}" "启动业务应用${process}失败，请用户检查启动脚本。" - START_APP_FLAG=1 - - fi - done - fi - if [[ "${start_step}" -eq 5 ]];then - if [[ ${START_APP_FLAG} -eq 1 || ${START_APP_COMM} -eq 1 ]]; then - for ((i = 0; i < "${length}"; i++)); do - process=${app_list[$i]} - process=$(echo ${process} | awk '{gsub(/^\s+|\s+$/, "");print}') - ps_result="$(check_process ${process})" - if [[ "${ps_result}" != "" ]]; then - write_messages c 34 "${start_step}" "业务应用${process}启动完成。" - fi - if [[ "${ps_result}" == "" ]]; then - write_messages e 0 "应用启动失败请检查应用启动命令" - exit 1 - fi - done - fi - fi - IFS="${OLD_IFS}" -} - -start_performance_test(){ - # 调用压力测试命令进行压力测试。 - PERF_TEST_FLAG=0 - export STRESS_CMD - if [[ $STRESS_CMD -eq 1 ]]; then - write_messages i 0 7 "启动压力测试工具" - if [[ ${TEST_TOOL_COMM} -eq 0 ]];then - OLD_IFS="${IFS}" - IFS=',' - read -r -a start_performance_scripts <<< "$start_performance_scripts" - length=${#start_performance_scripts[@]} - for ((i = 0; i < "${length}"; i++)); do - start_script=${start_performance_scripts[$i]} - pattern='&$' - if [[ $start_script =~ $pattern ]];then - eval "${start_script}" >> ./log/"${app_log_file}" 2>&1 - PID=$! - cd "${CURRENT_PATH}"||exit - else - eval "(${start_script})&" >> ./log/"${app_log_file}" 2>&1 - PID=$! - cd "${CURRENT_PATH}"||exit - fi - - if ! ps -fp "${PID}"> /dev/null; then - write_messages e 0 7 "调用命令${start_script}启动测试工具失败" - HAND_START_TEST=1 - PERF_TEST_FLAG=0 - else - performance_test_pid_array[$i]="${PID}" - PERF_TEST_FLAG=1 - fi - done - IFS="${OLD_IFS}" - fi - if [[ ${TEST_TOOL_COMM} -eq 1 || ${PERF_TEST_FLAG} -eq 0 ]];then - write_messages c 31 7 "启动压力测试工具失败，请确认启动命令" - exit 1 - fi - else - write_messages i 0 7 "用户未提供启动压力测试工具命令" - fi -} - -reliablity_test(){ - # 可靠性测试 - OLD_IFS="${IFS}" - IFS=',' - read -r -a app_list <<< "$application_names" - read -r -a start_comm_list <<< "$start_app_commands" - SLEEP_TIME=20 - RELIABLE_TEST_FLAG=0 - - for pid in "${performance_test_pid_array[@]}" - do - # 当前pid的所有子进程 - sub_pids=$(pgrep -P ${pid}) - for subpid in ${sub_pids}; - do - kill -9 "${subpid}" >>./log/"${log_file}" 2>&1 - done - disown "${pid}" - kill -9 "${pid}" >>./log/"${log_file}" 2>&1 - if [[ $? -ne 0 ]] ; then - write_messages e 0 8 "压力测试工具停止失败" - exit 1 - fi - sleep "${SLEEP_TIME}" - done - - if [[ ${KUBERNETES_ENV} -eq 0 && ${HAND_START_APP} -eq 0 && ${START_APP_COMM} -eq 0 ]];then - length=${#app_list[@]} - for ((i = 0; i < "${length}"; i++)); - do - process=${app_list[$i]} - process=$(echo ${process} | awk '{gsub(/^\s+|\s+$/, "");print}') - ps_result="$(check_process ${process})" - if [[ "${ps_result}" != "" ]]; then - write_messages i 0 8 "可靠性测试前检查，业务应用进程${process}存在" - if ! kill_process ${process} >>./log/"${log_file}" 2>&1 ; then - write_messages e 0 8 "可靠性测试，执行强制杀死进程${process}报错，可靠性测试失败。" - RELIABLE_TEST_FLAG=1 - else - write_messages i 0 8 "可靠性测试，执行强制杀死进程${process}" - fi - else - write_messages e 31 8 "可靠性测试前，业务应用${process}已停止，可靠性测试失败。" - RELIABLE_TEST_FLAG=1 - fi - done - sleep "${SLEEP_TIME}" - if [[ ${RELIABLE_TEST_FLAG} -eq 0 ]]; then - start_app 8 - for ((i = 0; i < "${length}"; i++)); - do - process=${app_list[$i]} - process=$(echo ${process} | awk '{gsub(/^\s+|\s+$/, "");print}') - ps_result="$(check_process ${process})" - if [[ "${ps_result}" != "" ]]; then - write_messages i 0 8 "可靠性测试，业务应用${process}启动完成。可靠性测试成功" - else - write_messages e 31 8 "可靠性测试，业务应用${process}启动失败,可靠性测试失败。" - fi - done - fi - elif [[ ${KUBERNETES_ENV} -eq 1 ]]; then - write_messages c 31 8 "业务应用是Kubernetes集群，不支持该用例测试。" - else - write_messages c 31 8 "业务应用是用户手动启动，不支持该用例测试。" - fi - IFS="${OLD_IFS}" -} - - -notice_users - -if [[ "$(id -u)" -ne 0 ]]; then - write_messages c 31 0 "请使用root用户权限执行脚本，否则没有权限检查系统日志。" -fi - -# 1、检查配置文件是否正确 -write_messages c 34 1 "第 1 步：配置文件检查, 开始" -check_configuration -write_messages c 34 1 "第 1 步：配置文件检查, 完成" -# 初始化，创建采集目录树 -write_messages c 34 1 "第 2 步：软件依赖检查，开始" -create_result_dir - -# 0 环境准备，判断用户环境 -env_preparation -check_physical_system -# 判断鲲鹏开发套件是否有启动，启动则调用命令停止 -stop_or_start_kunpengdeveloper 1 - -message_start_time="$(tail -1 /var/log/${SYS_LOG_}| awk -F' ' '{for(i=1;i<=3;i++) printf $i OFS}')" -write_messages c 34 1 "第 2 步：软件依赖检查，完成" - -# 2、环境自检 -write_messages c 34 3 "第 3 步：测试环境自检, 开始" -sys_env_inspectation -write_messages c 34 3 "第 3 步：测试环境自检, 完成" - -write_messages c 34 4 "第 4 步：应用启动前CPU、内存、硬盘、网卡和功耗系统资源采集" -# 3、系统信息采集 -get_service_info - -# 4、idle_0采集 -# 4.1、检查应用进程，若存在则停止 -check_process_exits_stop 4 -# 4.2、idle_0采集 -get_performance 5 1 0 4 -write_messages c 34 4 "第 4 步：应用启动前采集结束" -sleep 10 -# 5、启动业务 -write_messages c 34 5 "第 5 步：启动业务应用，可通过./log目录info.log查看进度，如果长时间未进行下一步，请检查配置文件填写的启动命令执行后是否会返回。" -start_app 5 -get_ps_snapshot -write_messages c 34 5 "第 5 步：启动业务应用完成" - -# 6、安全检查 -write_messages c 34 6 "第 6 步：安全测试，进行应用端口扫描，可通过./log目录的nmap.log查看进度" -port_scan -validated_clamav_scan -validated_cvecheck -write_messages c 34 6 "第 6 步：安全测试结束" - -# 7.1、进行压力测试 -write_messages c 34 7 "第 7 步：进行业务压力下CPU、内存、硬盘和网卡系统资源采集" -if [[ ${HPC_CERTIFICATE} -eq 0 && ${STRESS_CMD} -eq 1 ]]; then - start_performance_test - # 7.2、压力测试采集 - sleep 2 - get_performance 5 3 1 7 - sleep 20 -fi - - -write_messages c 34 7 "第 7 步：压力测试采集结束." - - -# 8、可靠性测试 -write_messages c 34 8 "第 8 步：进行可靠性测试，强制KILL应用后正常启动测试" -if [[ ${HPC_CERTIFICATE} -eq 0 ]]; then - reliablity_test -fi -write_messages c 34 8 "第 8 步：可靠性测试结束." - -# 9、idle_1测试 -write_messages c 34 9 "第 9 步：应用停止后CPU、内存、硬盘、网卡和功耗系统资源采集" -check_process_exits_stop 9 -get_performance 5 1 2 9 -sleep 10 -write_messages c 34 9 "第 9 步：应用停止后资源采集结束" - -write_messages c 34 10 "第 10 步：测试采集数据打包" -if [[ ${HPC_CERTIFICATE} -eq 1 ]]; then - source ./hpc_linpack_test.sh - bash hpc_memory_bandwidth.sh -fi -stop_or_start_kunpengdeveloper 10 -message_end_time="$(tail -1 /var/log/${SYS_LOG_}| awk -F' ' '{for(i=1;i<=3;i++) printf $i OFS}')" -check_system_message "${message_start_time}" "${message_end_time}" -# 检查以上采集是否有出错 -check_error -tar_output -exit diff --git a/component/LkpTests/compatibility_help/json2html.py b/component/LkpTests/compatibility_help/json2html.py deleted file mode 100644 index 014fae2e16b537277de4847d32af2844160b6495..0000000000000000000000000000000000000000 --- a/component/LkpTests/compatibility_help/json2html.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import json - -COMPATIBILITY_REPORT_ENTRIES = ["id", "result", "reason", "evidence"] -COMPATIBILITY_REPORT_TEMPLATE_HOLDER = "/** compatibility-test-report **/" -COMPATIBILITY_JSON_NAME = "log.json" - - -def compatibility_result_to_html(): - compatibility_result = [] - compatibility_result.extend(COMPATIBILITY_REPORT_ENTRIES) - with open(os.path.join(os.getcwd(), "Chinese", COMPATIBILITY_JSON_NAME), "r") as file: - data = json.load(file) - - for item in data: - compatibility_result.extend([item.get(entry) for entry in COMPATIBILITY_REPORT_ENTRIES]) - - compatibility_report = json.dumps(compatibility_result) - with open(os.path.join(os.getcwd(), "template.html"), "r") as file: - html_lines = file.readlines() - res = [sub.replace(COMPATIBILITY_REPORT_TEMPLATE_HOLDER, compatibility_report) for sub in html_lines] - with open(os.path.join(os.getcwd(), "compatibility_report.html"), "w") as f: - f.writelines(res) - - -compatibility_result_to_html() \ No newline at end of file diff --git a/component/LkpTests/compatibility_help/report_result.sh b/component/LkpTests/compatibility_help/report_result.sh deleted file mode 100644 index 9f546ea0018864c1ed48ddaced4bfa141c2839f8..0000000000000000000000000000000000000000 --- a/component/LkpTests/compatibility_help/report_result.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# SourceCode build script -# Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. - -cd ${HOME}/.local/compatibility_testing/ -result=`python3 ${HOME}/.local/compatibility_testing/check_report_result.py` - -if [ "$result" == "-1" ]; then - exit 1 -fi \ No newline at end of file diff --git a/component/LkpTests/compatibility_help/template.html.bak b/component/LkpTests/compatibility_help/template.html.bak deleted file mode 100644 index 6784ee19592fc5777c4245b04400c9f0c0323920..0000000000000000000000000000000000000000 --- a/component/LkpTests/compatibility_help/template.html.bak +++ /dev/null @@ -1,296 +0,0 @@ - - - - - - Kunpeng DevKit - - - - - - - - - - - - - - - - - -
- -
-
-
-
- - - diff --git a/component/LkpTests/gem_dependencies.zip b/component/LkpTests/gem_dependencies.zip deleted file mode 100644 index 4c972352a77355171057493dae3e1b5a5618b187..0000000000000000000000000000000000000000 Binary files a/component/LkpTests/gem_dependencies.zip and /dev/null differ diff --git a/component/LkpTests/install.sh b/component/LkpTests/install.sh deleted file mode 100644 index 15fd2efdbd9b009f9d09c9fdfcb8bf4d7bf65688..0000000000000000000000000000000000000000 --- a/component/LkpTests/install.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -e - - -gems_name=(zeitwerk-2.6.5.gem unicode-display_width-2.5.0.gem tzinfo-2.0.5.gem tins-1.31.1.gem term-ansicolor-1.7.1.gem sync-0.5.0.gem -simplecov-rcov-0.3.1.gem simplecov-html-0.12.3.gem simplecov-0.21.2.gem simplecov_json_formatter-0.1.4.gem ruby-progressbar-1.11.0.gem -rubocop-ast-1.17.0.gem rubocop-1.12.1.gem rspec-support-3.12.0.gem rspec-mocks-3.12.0.gem rspec-expectations-3.12.0.gem -rspec-core-3.12.0.gem rspec-3.12.0.gem rexml-3.2.5.gem regexp_parser-2.6.0.gem rchardet-1.8.0.gem rainbow-3.1.1.gem public_suffix-4.0.7.gem -parser-3.1.2.1.gem parallel-1.22.1.gem minitest-5.15.0.gem i18n-1.12.0.gem gnuplot-2.6.2.gem git-1.7.0.gem docile-1.4.0.gem diff-lcs-1.5.0.gem -concurrent-ruby-1.1.10.gem ci_reporter-2.0.0.gem bundler-2.2.33.gem builder-3.2.4.gem ast-2.4.2.gem activesupport-6.1.7.gem) - -function main() { - lkp_tar=/tmp/devkitdependencies/lkp-tests.tar.gz - gem_zip=/tmp/devkitdependencies/gem_dependencies.zip - rm -rf "${HOME}"/.local/lkp-tests/ - if [[ ! -d /usr/share/gems/gems/gem_dependencies ]]; then - unzip -d /usr/share/gems/gems ${gem_zip} - fi - cd /usr/share/gems/gems/gem_dependencies - for each in "${gems_name[@]}"; do - gem install --local ${each} - done - mkdir -p "${HOME}"/.local/ - tar --no-same-owner -zxf ${lkp_tar} -C "${HOME}"/.local/ - cd "${HOME}"/.local/lkp-tests/ - chmod +x "${HOME}"/.local/lkp-tests/bin/lkp - make - - chmod 777 "${HOME}"/.local/lkp-tests/programs/compatibility-test/run - ln -s "${HOME}"/.local/lkp-tests/programs/compatibility-test/run "${HOME}"/.local/lkp-tests/tests/compatibility-test - - cd "${HOME}"/.local/lkp-tests/programs/compatibility-test/ - lkp split "${HOME}"/.local/lkp-tests/programs/compatibility-test/jobs/compatibility-test.yaml - -} -main "$@" \ No newline at end of file diff --git a/component/LkpTests/lkp_help/compatibility-test/jobs/compatibility-test.yaml b/component/LkpTests/lkp_help/compatibility-test/jobs/compatibility-test.yaml deleted file mode 100644 index b411ad12951cb5d32f79f0147f56f9905a6a818d..0000000000000000000000000000000000000000 --- a/component/LkpTests/lkp_help/compatibility-test/jobs/compatibility-test.yaml +++ /dev/null @@ -1,5 +0,0 @@ -suite: compatibility-test -category: functional - -file_path: ${HOME}/.local/compatibility_testing/Chinese/ -compatibility-test: diff --git a/component/LkpTests/lkp_help/compatibility-test/meta.yaml b/component/LkpTests/lkp_help/compatibility-test/meta.yaml deleted file mode 100644 index 7b114beafd4f5261d768ff35f5626980acd4f999..0000000000000000000000000000000000000000 --- a/component/LkpTests/lkp_help/compatibility-test/meta.yaml +++ /dev/null @@ -1,9 +0,0 @@ -metadata: - name: compatibility-test - summary: A program can run some basic tests - description: run compatibility test and generate the report - homepage: https://gitee.com/openeuler/devkit-pipeline -type: workload -depends: -params: -results: diff --git a/component/LkpTests/lkp_help/compatibility-test/run b/component/LkpTests/lkp_help/compatibility-test/run deleted file mode 100644 index 2b49a468207373d8f2d16381baca3b2feee2cecb..0000000000000000000000000000000000000000 --- a/component/LkpTests/lkp_help/compatibility-test/run +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e -ct_sh_path=${HOME}/.local/compatibility_testing/Chinese/compatibility_testing.sh -cloud_jar=${HOME}/.local/compatibility_testing/cloudTest.jar - -cd ${HOME}/.local/compatibility_testing/Chinese/ -sh $ct_sh_path - -java -jar $cloud_jar & -sleep 15 -jar_pid=$! -curl --location --request GET 'http://127.0.0.1:10037/api/v1/report?savePath=/'${HOME}'/.local/compatibility_testing/Chinese/log.json&file=/'${HOME}'/.local/compatibility_testing/Chinese/log.tar.gz' -kill -9 $jar_pid -cp -rf ${HOME}/.local/compatibility_testing/template.html.bak ${HOME}/.local/compatibility_testing/template.html -cd ${HOME}/.local/compatibility_testing/ -python3 ${HOME}/.local/compatibility_testing/json2html.py diff --git a/component/LkpTests/lkp_help/devkit-pipeline.patch b/component/LkpTests/lkp_help/devkit-pipeline.patch deleted file mode 100644 index 1ffab9165524ba06e2daea45758c24792cde5aa6..0000000000000000000000000000000000000000 --- a/component/LkpTests/lkp_help/devkit-pipeline.patch +++ /dev/null @@ -1,42 +0,0 @@ -diff --git a/lkp-exec/install b/lkp-exec/install -index 9d77c9c07..ed7e28c7e 100755 ---- a/lkp-exec/install -+++ b/lkp-exec/install -@@ -205,14 +205,6 @@ install_base_support() - { - echo "Installing ruby gems..." - -- if [ -n "$CHINA" ]; then -- setup_gem_china -- fi -- ( -- cd "$LKP_SRC" || exit -- bundle_install || exit -- ) || exit -- - create_lkp_user - create_lkp_dirs - } -@@ -234,7 +226,6 @@ create_host_config - if [[ -z "$1" ]]; then - install_base_support - all_scripts='lkp-tests makepkg' --else - make_wakeup - all_scripts= - fi -diff --git a/sbin/install-dependencies.sh b/sbin/install-dependencies.sh -index 5e41554bd..b41ba4f94 100755 ---- a/sbin/install-dependencies.sh -+++ b/sbin/install-dependencies.sh -@@ -1,10 +1,6 @@ - #!/usr/bin/env bash - - INSTALL_SUCCESS_FILE=/tmp/.lkp-install-dependencies-success --test -e $INSTALL_SUCCESS_FILE && { -- echo "$0: already installed: $INSTALL_SUCCESS_FILE" -- exit 0 --} - - SCRIPT_DIR=$(cd $(dirname $0); pwd -P) - export LKP_SRC=$(dirname $SCRIPT_DIR) diff --git a/component/LkpTests/lkp_help/lkp_help.sh b/component/LkpTests/lkp_help/lkp_help.sh deleted file mode 100644 index 3d650bdfd489acfe4141339e2262db0ec0f3064e..0000000000000000000000000000000000000000 --- a/component/LkpTests/lkp_help/lkp_help.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# SourceCode build script -# Copyright: Copyright (c) Huawei Technologies Co., Ltd. All rights reserved. - -current_dir=$(cd $(dirname $0); pwd) - -function main() { - upload_path=$1 - cd ${current_dir} - if [[ -d ${current_dir}/lkp-tests ]]; then - rm -rf ${current_dir}/lkp-tests - fi - - wget -c https://gitee.com/jerry-553/lkp_test_devkitpipeline/releases/download/lkp-all-resource/lkp-tests.tar.gz - if [[ "$?" -ne "0" ]]; then - exit 1 - fi - - echo "/bin/cp -rf lkp-tests.tar.gz ${upload_path}" - /bin/cp -rf lkp-tests.tar.gz ${upload_path} -} - -main "$@" diff --git a/component/NonInvasiveSwitching/install.sh b/component/NonInvasiveSwitching/install.sh index b49e8d579c493250b2b6c19c4752ad18bf7239e6..c5a1c2e9818ae1b70b0cfa9fcedbf30fc30cd456 100644 --- a/component/NonInvasiveSwitching/install.sh +++ b/component/NonInvasiveSwitching/install.sh @@ -5,15 +5,15 @@ function main() { set -x "${HOME}"/.local/BiShengCompiler-3.2.0-aarch64-linux/bin/clang "$@" -mcpu=tsv110 -O3 -g EOF - cat > "${HOME}"/.local/wrap-bin/clang++ <<'EOF' + cat > "${HOME}"/.local/wrap-bin/clang++ <<'EOF' set -x "${HOME}"/.local/BiShengCompiler-3.2.0-aarch64-linux/bin/clang++ "$@" -mcpu=tsv110 -O3 -g EOF - cat > "${HOME}"/.local/wrap-bin/gcc <<'EOF' + cat > "${HOME}"/.local/wrap-bin/gcc <<'EOF' set -x "${HOME}"/.local/BiShengCompiler-3.2.0-aarch64-linux/bin/clang "$@" -mcpu=tsv110 -O3 -g EOF - cat > "${HOME}"/.local/wrap-bin/g++ <<'EOF' + cat > "${HOME}"/.local/wrap-bin/g++ <<'EOF' set -x "${HOME}"/.local/BiShengCompiler-3.2.0-aarch64-linux/bin/clang++ "$@" -mcpu=tsv110 -O3 -g EOF diff --git a/component/build_component.sh b/component/build_component.sh index 37106d1a0d0158cf5503d4c5227a427bfc5e4664..8a4de57bba47bd2c9e7c4f2822418e5775addba1 100644 --- a/component/build_component.sh +++ b/component/build_component.sh @@ -10,16 +10,12 @@ final_component_dir=${build_dir}/component umask 077 -function build_lkp_tests() { - # 单独处理LkpTests - mkdir -p "${final_component_dir}"/LkpTests +function build_devkit_tester() { + mkdir -p "${final_component_dir}"/DevKitTester + cp "${current_dir}"/DevKitTester/install.sh "${final_component_dir}"/DevKitTester + cp "${current_dir}"/DevKitTester/check_install_result.sh "${final_component_dir}"/DevKitTester - cp -rf "${current_dir}"/LkpTests/install.sh "${final_component_dir}"/LkpTests - cp -rf "${current_dir}"/LkpTests/check_install_result.sh "${final_component_dir}"/LkpTests -} - -function build_devkit_distributor() { - bash "${current_dir}"/DevKitTester/build_devkit_tester.sh + # bash "${current_dir}"/DevKitTester/build_devkit_tester.sh } function build_devkit_installer() { @@ -34,15 +30,16 @@ function main() { mkdir -p "${final_component_dir}" component_arrays=( - "BiShengCompiler" "BiShengJDK8" "BiShengJDK17" "GCCforOpenEuler" "OpenEulerMirrorISO" "CompatibilityTesting" "NonInvasiveSwitching" "A-FOT" - "DevKitCLI" "clamAV" + "BiShengCompiler" "BiShengJDK8" "BiShengJDK17" "GCCforOpenEuler" + "CompatibilityTesting" "NonInvasiveSwitching" "A-FOT" + "DevKitCLI" "ClamAV" ) for element in "${component_arrays[@]}"; do cp -rf "${current_dir}/${element}" "${final_component_dir}" done - build_lkp_tests - build_devkit_distributor + rm -fr "${final_component_dir}/{DevKitTester, DevKitWeb, OpenEulerMirrorISO}" + build_devkit_tester build_devkit_installer } diff --git a/tools/install_dependency/build_download.sh b/tools/download_and_deploy/build_download.sh similarity index 88% rename from tools/install_dependency/build_download.sh rename to tools/download_and_deploy/build_download.sh index 0201b05bd23da8ef44de2b7f62a2686b813091dd..ed51b8e3dde2359930207dfa2cce9a9fe430a8b1 100644 --- a/tools/install_dependency/build_download.sh +++ b/tools/download_and_deploy/build_download.sh @@ -8,7 +8,7 @@ project_dir=$(dirname $(dirname "${current_dir}")) umask 077 -build_dir=${project_dir}/build/download_dependency +build_dir=${project_dir}/build/download_tool rm -rf "${build_dir}" mkdir -p "${build_dir}" diff --git a/tools/install_dependency/build_install.sh b/tools/download_and_deploy/build_install.sh similarity index 91% rename from tools/install_dependency/build_install.sh rename to tools/download_and_deploy/build_install.sh index f2b7a7fd13ad0cc80d425e02eaf4e76dfbee4acf..e9b38a5b236f626e659d97be7fda1e895fd6aced 100644 --- a/tools/install_dependency/build_install.sh +++ b/tools/download_and_deploy/build_install.sh @@ -8,7 +8,7 @@ project_dir=$(dirname $(dirname "${current_dir}")) umask 077 -build_dir=${project_dir}/build/install_dependency +build_dir=${project_dir}/build/deploy_tool rm -rf "${build_dir}" mkdir -p "${build_dir}" diff --git a/tools/install_dependency/config/machine.yaml b/tools/download_and_deploy/config/machine.yaml similarity index 66% rename from tools/install_dependency/config/machine.yaml rename to tools/download_and_deploy/config/machine.yaml index 15e19addaabc08bae245d3a1b3722dfec8c055df..b5d24a5d07b2ee3f8ee8c1494f61e9505e3c0d23 100644 --- a/tools/install_dependency/config/machine.yaml +++ b/tools/download_and_deploy/config/machine.yaml @@ -4,17 +4,19 @@ scanner: - 192.168.0.1 - 192.168.0.2 - 192.168.0.3 +c_builder_gcc: + - 192.168.0.2 +c_builder_bisheng_compiler: + - 192.168.0.2 java_builder_jdk8: - 192.168.0.1 java_builder_jdk17: - 192.168.0.1 -c_cpp_builder: - - 192.168.0.2 -c_cpp_builder_gcc: - - 192.168.0.2 -c_cpp_builder_bisheng_compiler: - - 192.168.0.2 executor: - 192.168.0.1 +compatibility: + - 192.168.0.1 devkit: - - 192.168.0.4 \ No newline at end of file + - 192.168.0.3 +clamav: + - 192.168.0.3 \ No newline at end of file diff --git a/tools/install_dependency/src/__init__.py b/tools/download_and_deploy/src/__init__.py similarity index 100% rename from tools/install_dependency/src/__init__.py rename to tools/download_and_deploy/src/__init__.py diff --git a/tools/install_dependency/src/deploy/__init__.py b/tools/download_and_deploy/src/component_install/__init__.py similarity index 100% rename from tools/install_dependency/src/deploy/__init__.py rename to tools/download_and_deploy/src/component_install/__init__.py diff --git a/tools/download_and_deploy/src/component_install/a_fot_deploy.py b/tools/download_and_deploy/src/component_install/a_fot_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..6b338ffbaf78581fc6f029eae884fc3d208685aa --- /dev/null +++ b/tools/download_and_deploy/src/component_install/a_fot_deploy.py @@ -0,0 +1,17 @@ +import os +import constant +from component_install.deploy_base import DeployBase +from utils import CHECK_PERF_AVAILABLE_CMD + + +class AFotDeploy(DeployBase): + component_name = "A-FOT" + remote_file_list = [ + os.path.join("/tmp", constant.DEPENDENCY_DIR, "a-fot.tar.gz"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "A-FOT" + "install.sh"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "A-FOT" + "check_install_result.sh"), + ] + + @classmethod + def before_upload(cls, machine, sftp_client, ssh_client): + cls._remote_exec_command(machine.ip, ssh_client, CHECK_PERF_AVAILABLE_CMD) diff --git a/tools/download_and_deploy/src/component_install/bisheng_compiler_deploy.py b/tools/download_and_deploy/src/component_install/bisheng_compiler_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..d3f05c01185eb60558aa5e57851adb12f974ecf8 --- /dev/null +++ b/tools/download_and_deploy/src/component_install/bisheng_compiler_deploy.py @@ -0,0 +1,12 @@ +import os +import constant +from component_install.deploy_base import DeployBase + + +class BiShengCompilerDeploy(DeployBase): + component_name = "BiShengCompiler" + remote_file_list = [ + os.path.join("/tmp", constant.DEPENDENCY_DIR, "BiShengCompiler-3.2.0-aarch64-linux.tar.gz"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "BiShengCompiler" + "install.sh"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "BiShengCompiler" + "check_install_result.sh"), + ] diff --git a/tools/download_and_deploy/src/component_install/bisheng_jdk17_deploy.py b/tools/download_and_deploy/src/component_install/bisheng_jdk17_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..cb52f379d4b731e12dace92b82b651da4df8c2ce --- /dev/null +++ b/tools/download_and_deploy/src/component_install/bisheng_jdk17_deploy.py @@ -0,0 +1,12 @@ +import os +import constant +from component_install.deploy_base import DeployBase + + +class BiShengJDK17Deploy(DeployBase): + component_name = "BiShengJDK17" + remote_file_list = [ + os.path.join("/tmp", constant.DEPENDENCY_DIR, "bisheng-jdk-17.0.10-linux-aarch64.tar.gz"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "BiShengJDK17" + "install.sh"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "BiShengJDK17" + "check_install_result.sh"), + ] diff --git a/tools/download_and_deploy/src/component_install/bisheng_jdk8_deploy.py b/tools/download_and_deploy/src/component_install/bisheng_jdk8_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..26a9b0705f2d38feec23aa525f9a60bb52fa9093 --- /dev/null +++ b/tools/download_and_deploy/src/component_install/bisheng_jdk8_deploy.py @@ -0,0 +1,12 @@ +import os +import constant +from component_install.deploy_base import DeployBase + + +class BiShengJDK8Deploy(DeployBase): + component_name = "BiShengJDK8" + remote_file_list = [ + os.path.join("/tmp", constant.DEPENDENCY_DIR, "bisheng-jdk-8u402-linux-aarch64.tar.gz"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "BiShengJDK8" + "install.sh"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "BiShengJDK8" + "check_install_result.sh"), + ] diff --git a/tools/download_and_deploy/src/component_install/clamav_deploy.py b/tools/download_and_deploy/src/component_install/clamav_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..41e9bb6d280f487ab731e226b8b8dd8c4769efcd --- /dev/null +++ b/tools/download_and_deploy/src/component_install/clamav_deploy.py @@ -0,0 +1,27 @@ +import os +import timeout_decorator + +import constant +from component_install.deploy_base import DeployBase, LOGGER +from utils import PROMPT_MAP, MKDIR_TMP_DEVKITDEPENDENCIES_CMD + + +class ClamAVDeploy(DeployBase): + component_name = "ClamAV" + remote_file_list = [ + os.path.join("/tmp", constant.DEPENDENCY_DIR, "ClamAV" + "install.sh"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "ClamAV" + "check_install_result.sh"), + ] + + @classmethod + def upload(cls, machine, sftp_client, ssh_client): + cls._remote_exec_command(machine.ip, ssh_client, MKDIR_TMP_DEVKITDEPENDENCIES_CMD) + + cmd = "yum install clamav -y" + ip = machine.ip + try: + cls._remote_exec_command(ip, ssh_client, cmd) + except (timeout_decorator.TimeoutError, OSError, IOError) as e: + LOGGER.error(f"Remote machine {ip} install {cls.component_name} failed. " + f"Exec '{cmd}' failed Please run this command in this machine.") + raise OSError(PROMPT_MAP.get(cmd, f"Remote machine {ip} exec '{cmd}' failed.")) diff --git a/tools/download_and_deploy/src/component_install/compatibility_test_deploy.py b/tools/download_and_deploy/src/component_install/compatibility_test_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..6b00b777eddb8362f6a0523d59a875ea1e12fa3e --- /dev/null +++ b/tools/download_and_deploy/src/component_install/compatibility_test_deploy.py @@ -0,0 +1,12 @@ +import os +import constant +from component_install.deploy_base import DeployBase + + +class CompatibilityDeploy(DeployBase): + component_name = "CompatibilityTesting" + remote_file_list = [ + os.path.join("/tmp", constant.DEPENDENCY_DIR, "compatibility_testing.tar.gz"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "CompatibilityTesting" + "install.sh"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "CompatibilityTesting" + "check_install_result.sh"), + ] diff --git a/tools/download_and_deploy/src/component_install/deploy_base.py b/tools/download_and_deploy/src/component_install/deploy_base.py new file mode 100644 index 0000000000000000000000000000000000000000..5aa63d33336aee838d85054a4e9384f0c7eb2812 --- /dev/null +++ b/tools/download_and_deploy/src/component_install/deploy_base.py @@ -0,0 +1,133 @@ +import logging +import os +import socket +import paramiko +import timeout_decorator + +import constant + +from download.download_utils import component_collection_map +from utils import (base_path, MKDIR_TMP_DEVKITDEPENDENCIES_CMD, + CHECK_TAR_AVAILABLE_CMD, CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE, CHECK_SUDO_PERMISSION, + PROMPT_MAP) + +LOGGER = logging.getLogger("deploy_tool") + + +class DeployBase: + component_name = "component_name" + sudo = False + remote_file_list = [] + + @classmethod + def install_handle(cls, machine, sftp_client, ssh_client): + cls.before_upload(machine, sftp_client, ssh_client) + cls.upload(machine, sftp_client, ssh_client) + cls.install(machine, sftp_client, ssh_client) + cls.after_install(machine, sftp_client, ssh_client) + + @classmethod + def before_upload(cls, machine, sftp_client, ssh_client): + if cls.sudo: + cls._remote_exec_command(machine.ip, ssh_client, CHECK_SUDO_PERMISSION) + + @classmethod + def upload(cls, machine, sftp_client, ssh_client): + cls._remote_exec_command(machine.ip, ssh_client, CHECK_TAR_AVAILABLE_CMD) + cls._remote_exec_command(machine.ip, ssh_client, MKDIR_TMP_DEVKITDEPENDENCIES_CMD) + cls._remote_exec_command(machine.ip, ssh_client, CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE) + + # 上传 .tar.gz 文件 + url_and_save_path = component_collection_map.get(cls.component_name) + local_file = url_and_save_path.get("save_path") + remote_file = os.path.abspath(os.path.join('/tmp', constant.DEPENDENCY_DIR, local_file.split('/')[-1])) + LOGGER.debug(f"Transport local_file: {local_file} to remote machine {machine.ip} " + f"remote_file: {remote_file}") + + # cls.remote_file_list.append(remote_file) + sftp_client.put(localpath=f"{local_file}", remotepath=f"{remote_file}") + + @classmethod + def install(cls, machine, sftp_client, ssh_client): + shell_file_list = ["install.sh", "check_install_result.sh"] + + # 上传并执行 安装脚本, 校验安装结果脚本 + install_result = "" + for shell_file in shell_file_list: + sh_file_local_path = os.path.join(base_path("component"), cls.component_name, shell_file) + sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, cls.component_name + shell_file) + + sh_cmd = f"bash {sh_file_remote_path}" + + execute_output = ( + cls._transport_shell_file_and_execute( + ip=machine.ip, + sftp_client=sftp_client, + ssh_client=ssh_client, + sh_file_local_path=sh_file_local_path, + sh_file_remote_path=sh_file_remote_path, + sh_cmd=sh_cmd + )) + # cls.remote_file_list.append(sh_file_remote_path) + if shell_file == shell_file_list[1]: + install_result = execute_output + + if install_result == "true": + LOGGER.info(f"Remote machine {machine.ip} install {cls.component_name} success.") + machine.component_dict[cls.component_name] = "install success." + else: + LOGGER.error(f"Remote machine {machine.ip} install {cls.component_name} failed.") + machine.component_dict[cls.component_name] = "install failed." + + @classmethod + def after_install(cls, machine, sftp_client, ssh_client): + cls._clear_tmp_file_at_remote_machine(machine.ip, ssh_client, cls.remote_file_list) + + @classmethod + @timeout_decorator.timeout(300) + def _remote_exec_command(cls, ip, ssh_client, cmd): + try: + stdin, stdout, stderr = ssh_client.exec_command(cmd, timeout=299) + except (paramiko.ssh_exception.SSHException, socket.timeout) as e: + LOGGER.error(f"Remote machine {ip} exec '{cmd}' failed Please run this command in this machine.") + raise OSError(PROMPT_MAP.get(cmd, f"Remote machine {ip} exec '{cmd}' failed.")) + exit_status = stdout.channel.recv_exit_status() + if exit_status == 0: + LOGGER.debug(f"Remote machine {ip} exec '{cmd}' success.") + else: + LOGGER.error(f"Remote machine {ip} exec '{cmd}' failed. Please run this command in this machine.") + raise OSError(PROMPT_MAP.get(cmd, f"Remote machine {ip} exec '{cmd}' failed.")) + + @classmethod + def _transport_shell_file_and_execute(cls, ip, sftp_client, ssh_client, + sh_file_local_path, sh_file_remote_path, + sh_cmd): + if not os.path.exists(sh_file_local_path): + LOGGER.error(f"{sh_file_local_path} not exists.") + raise FileNotFoundError(f"local file {sh_file_local_path} not exists.") + + LOGGER.debug(f"Transport local_file: {sh_file_local_path} to remote machine {ip} " + f"remote_file: {sh_file_remote_path}") + sftp_client.put(localpath=sh_file_local_path, remotepath=sh_file_remote_path) + + try: + stdin, stdout, stderr = ssh_client.exec_command(sh_cmd) + except (paramiko.ssh_exception.SSHException, socket.timeout) as e: + LOGGER.error(f"Remote machine {ip} exec '{sh_cmd}' failed. {str(e)}") + raise OSError(f"Remote machine {ip} exec '{sh_cmd}' failed. {str(e)}") + output = stdout.read().decode().strip() + LOGGER.info(f"Remote machine {ip} '{sh_cmd}' output: {output}") + return output + + @classmethod + def _clear_tmp_file_at_remote_machine(cls, ip, ssh_client, remote_file_list): + LOGGER.debug(f"Clear tmp file at remote machine {ip}") + for remote_file in remote_file_list: + try: + remote_file = os.path.realpath(remote_file) + if not remote_file.startswith(os.path.join("/tmp", constant.DEPENDENCY_DIR)): + continue + LOGGER.debug(f"Delete tmp file at remote machine {ip}: {remote_file}") + ssh_client.exec_command(f"rm -fr {remote_file}") + except Exception as e: + LOGGER.debug(str(e)) diff --git a/tools/download_and_deploy/src/component_install/devkitcli_deploy.py b/tools/download_and_deploy/src/component_install/devkitcli_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..5c15ce2e685a609b5c35b188901ed4ba510e33b0 --- /dev/null +++ b/tools/download_and_deploy/src/component_install/devkitcli_deploy.py @@ -0,0 +1,13 @@ +import os +import constant +from component_install.deploy_base import DeployBase + + +class DevkitCLIDeploy(DeployBase): + component_name = "DevKitCLI" + remote_file_list = [ + os.path.join("/tmp", constant.DEPENDENCY_DIR, "DevKit-CLI-24.0.RC1-Linux-Kunpeng.tar.gz"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "DevKitCLI" + "install.sh"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "DevKitCLI" + "check_install_result.sh"), + ] + sudo = True diff --git a/tools/download_and_deploy/src/component_install/devkittester_deploy.py b/tools/download_and_deploy/src/component_install/devkittester_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..e735f7e2d204266c2e9838fd19fdea6e7a75855c --- /dev/null +++ b/tools/download_and_deploy/src/component_install/devkittester_deploy.py @@ -0,0 +1,12 @@ +import os +import constant +from component_install.deploy_base import DeployBase + + +class DevkitTesterDeploy(DeployBase): + component_name = "DevKitTester" + remote_file_list = [ + os.path.join("/tmp", constant.DEPENDENCY_DIR, "devkit_tester.tar.gz"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "DevKitTester" + "install.sh"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "DevKitTester" + "check_install_result.sh"), + ] diff --git a/tools/download_and_deploy/src/component_install/devkitweb_deploy.py b/tools/download_and_deploy/src/component_install/devkitweb_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..8cf7ba1ee29b28584c028b61d617b23d43efed35 --- /dev/null +++ b/tools/download_and_deploy/src/component_install/devkitweb_deploy.py @@ -0,0 +1,41 @@ +import os +import subprocess + +from download.download_utils import component_collection_map +from component_install.deploy_base import DeployBase, LOGGER +from utils import base_path, CHECK_OPT_SPACE_SUFFICIENT_FOR_PACKAGE, CHECK_SUDO_PERMISSION, CHECK_OPT_WRITE_PERMISSION + + +class DevkitWebDeploy(DeployBase): + component_name = "DevKitWeb" + remote_file_list = [ + os.path.join("/opt", "DevKit-All-24.0.RC1-Linux-Kunpeng.tar.gz") + ] + + @classmethod + def before_upload(cls, machine, sftp_client, ssh_client): + cls._remote_exec_command(machine.ip, ssh_client, CHECK_SUDO_PERMISSION) + cls._remote_exec_command(machine.ip, ssh_client, CHECK_OPT_WRITE_PERMISSION) + cls._remote_exec_command(machine.ip, ssh_client, CHECK_OPT_SPACE_SUFFICIENT_FOR_PACKAGE) + + @classmethod + def upload(cls, machine, sftp_client, ssh_client): + url_and_save_path = component_collection_map.get(cls.component_name) + local_file = url_and_save_path.get("save_path") + remote_file = os.path.abspath(os.path.join('/opt', local_file.split('/')[-1])) + LOGGER.debug(f"Transport local_file: {local_file} to remote machine {machine.ip} " + f"remote_file: {remote_file}") + + # cls.remote_file_list.append(remote_file) + sftp_client.put(localpath=f"{local_file}", remotepath=f"{remote_file}") + + @classmethod + def install(cls, machine, sftp_client, ssh_client): + local_file = component_collection_map.get(cls.component_name).get("save_path") + cmd = f"{os.path.join(base_path('component'), cls.component_name, 'devkit_installer')} " \ + f"-i {machine.ip} -u {machine.user} -p {machine.pkey} -paname {local_file.split('/')[-1]} --debug" + LOGGER.debug(f"Executing command: {cmd}") + result = subprocess.run(cmd.split(' '), + capture_output=False, shell=False, stderr=subprocess.STDOUT) + if result.returncode == 0: + machine.component_dict[cls.component_name] = "install success." diff --git a/tools/download_and_deploy/src/component_install/gcc_deploy.py b/tools/download_and_deploy/src/component_install/gcc_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..16d3ba9ebd58db3c9c09b3e5a0c32f6eb1fa9ff3 --- /dev/null +++ b/tools/download_and_deploy/src/component_install/gcc_deploy.py @@ -0,0 +1,12 @@ +import os +import constant +from component_install.deploy_base import DeployBase + + +class GccDeploy(DeployBase): + component_name = "GCCforOpenEuler" + remote_file_list = [ + os.path.join("/tmp", constant.DEPENDENCY_DIR, "gcc-10.3.1-2023.12-aarch64-linux.tar.gz"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "GCCforOpenEuler" + "install.sh"), + os.path.join("/tmp", constant.DEPENDENCY_DIR, "GCCforOpenEuler" + "check_install_result.sh"), + ] diff --git a/tools/download_and_deploy/src/component_install/switch_deploy.py b/tools/download_and_deploy/src/component_install/switch_deploy.py new file mode 100644 index 0000000000000000000000000000000000000000..3b05e0cec95af8e3e6e5c5a94b0fb30ddc433095 --- /dev/null +++ b/tools/download_and_deploy/src/component_install/switch_deploy.py @@ -0,0 +1,16 @@ +import os +import constant +from component_install.deploy_base import DeployBase +from utils import MKDIR_TMP_DEVKITDEPENDENCIES_CMD + + +class SwitchDeploy(DeployBase): + component_name = "NonInvasiveSwitching" + remote_file_list = [ + os.path.join("/tmp/", constant.DEPENDENCY_DIR, "NonInvasiveSwitching" + "install.sh"), + os.path.join("/tmp/", constant.DEPENDENCY_DIR, "NonInvasiveSwitching" + "check_install_result.sh"), + ] + + @classmethod + def upload(cls, machine, sftp_client, ssh_client): + cls._remote_exec_command(machine.ip, ssh_client, MKDIR_TMP_DEVKITDEPENDENCIES_CMD) diff --git a/tools/install_dependency/src/constant.py b/tools/download_and_deploy/src/constant.py similarity index 33% rename from tools/install_dependency/src/constant.py rename to tools/download_and_deploy/src/constant.py index 74d09c3eca1c7ce4470edf8c8adb1a244d8e4f1b..96edcdad6ba1f772b579c689968fd62ddb84059a 100644 --- a/tools/install_dependency/src/constant.py +++ b/tools/download_and_deploy/src/constant.py @@ -1,32 +1,55 @@ USER = "user" PKEY = "pkey" PASSWORD = "password" + SCANNER = "scanner" + +C_BUILDER = "c_builder" +C_BUILDER_GCC = "c_builder_gcc" +C_BUILDER_BISHENG_COMPILER = "c_builder_bisheng_compiler" + +JAVA_BUILDER = "java_builder" JAVA_BUILDER_JDK8 = "java_builder_jdk8" JAVA_BUILDER_JDK17 = "java_builder_jdk17" -C_BUIDLER = "c_cpp_builder" -C_BUIDLER_GCC = "c_cpp_builder_gcc" -C_BUIDLER_BISHENG_COMPILER = "c_cpp_builder_bisheng_compiler" + EXECUTOR = "executor" +COMPATIBILITY = "compatibility" +TESTER = "tester" + DEVKIT = "devkit" +CLAMAV = "clamav" + MACHINE = "machine" DEPENDENCY_FILE = "devkitdependencies.tar.gz" DEPENDENCY_DIR = "devkitdependencies" -INSTRUCTION = "instruction" + +ROLE_LIST = [ + SCANNER, + C_BUILDER, C_BUILDER_GCC, C_BUILDER_BISHENG_COMPILER, + JAVA_BUILDER, JAVA_BUILDER_JDK8, JAVA_BUILDER_JDK17, + EXECUTOR, COMPATIBILITY, TESTER, + DEVKIT, + CLAMAV, +] ROLE_COMPONENT = { - SCANNER: ["BiShengJDK17", "DevKitCLI"], - C_BUIDLER: ["GCCforOpenEuler", "BiShengCompiler", "BiShengJDK17", "A-FOT", "NonInvasiveSwitching"], - C_BUIDLER_GCC: ["GCCforOpenEuler", "BiShengJDK17", "A-FOT"], - C_BUIDLER_BISHENG_COMPILER: ["BiShengCompiler", "BiShengJDK17", "NonInvasiveSwitching"], - JAVA_BUILDER_JDK8: ["BiShengJDK8", "BiShengJDK17"], + SCANNER: ["DevKitCLI"], + C_BUILDER: ["GCCforOpenEuler", "BiShengCompiler", "A-FOT", "NonInvasiveSwitching"], + C_BUILDER_GCC: ["GCCforOpenEuler", "A-FOT"], + C_BUILDER_BISHENG_COMPILER: ["BiShengCompiler", "NonInvasiveSwitching"], + JAVA_BUILDER: ["BiShengJDK8", "BiShengJDK17"], + JAVA_BUILDER_JDK8: ["BiShengJDK8"], JAVA_BUILDER_JDK17: ["BiShengJDK17"], - EXECUTOR: ["BiShengJDK17", "LkpTests", "CompatibilityTesting", "DevKitTester"], - DEVKIT: ["DevKitWeb"] + EXECUTOR: ["CompatibilityTesting"], + COMPATIBILITY: ["CompatibilityTesting"], + TESTER: [ + # "DevKitTester" + ], + DEVKIT: ["DevKitWeb"], + CLAMAV: ["ClamAV"], } -ROLE_LIST = [SCANNER, C_BUIDLER, C_BUIDLER_GCC, C_BUIDLER_BISHENG_COMPILER, JAVA_BUILDER_JDK8, JAVA_BUILDER_JDK17, EXECUTOR, DEVKIT] FILE = "file" SHA256 = "sha256" diff --git a/tools/install_dependency/src/download/__init__.py b/tools/download_and_deploy/src/deploy/__init__.py similarity index 100% rename from tools/install_dependency/src/download/__init__.py rename to tools/download_and_deploy/src/deploy/__init__.py diff --git a/tools/install_dependency/src/deploy/deploy_command_line.py b/tools/download_and_deploy/src/deploy/deploy_command_line.py similarity index 75% rename from tools/install_dependency/src/deploy/deploy_command_line.py rename to tools/download_and_deploy/src/deploy/deploy_command_line.py index 9e3150503faf70e46272c30ea9babb40ed547b83..c5b4fbe62dbab13fbec4ea0e8d0ea9235b16a5e9 100644 --- a/tools/install_dependency/src/deploy/deploy_command_line.py +++ b/tools/download_and_deploy/src/deploy/deploy_command_line.py @@ -12,16 +12,11 @@ class CommandLine: def add_options(cls, parser): parser.add_argument("-f", "--config", action="store", dest="yaml_path", default=DEFAULT_YAML_PATH, help="Assign yaml config file path. Default path is 'machine.yaml' in current directory.") - parser.add_argument("-iso", action="store", dest="iso_path", default=None, - help="Assign prepared iso file path.") - parser.add_argument("--debug", action="store_true", dest="debug", default=True, - help="Open debug log. Default is True.") parser.add_argument("--silent", action="store_true", dest="silent", default=False, help="Close debug log.") @classmethod def process_args(cls, args): cls.yaml_path = args.yaml_path if args.yaml_path and args.yaml_path != "./" else DEFAULT_YAML_PATH - cls.iso_path = args.iso_path cls.silent = args.silent return cls.yaml_path diff --git a/tools/install_dependency/src/deploy_main.py b/tools/download_and_deploy/src/deploy_main.py similarity index 85% rename from tools/install_dependency/src/deploy_main.py rename to tools/download_and_deploy/src/deploy_main.py index 3069c2a731707684f6a9b93ce71c190c90999ca7..8d131165b4f021ee28515aa7a3909ad960981416 100644 --- a/tools/install_dependency/src/deploy_main.py +++ b/tools/download_and_deploy/src/deploy_main.py @@ -3,8 +3,6 @@ import subprocess import sys import logging - -import constant from log import config_logging from deploy.deploy_command_line import process_command_line, CommandLine @@ -13,10 +11,11 @@ from handler.base_yaml_check import BaseCheck from handler.connect_check import ConnectCheck from handler.gather_package import GatherPackage from handler.install_package import InstallPackage +from handler.result_report import ResultReport from utils import read_yaml_file -LOGGER = logging.getLogger("install_dependency") -PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage()] +LOGGER = logging.getLogger("deploy_tool") +PIPELINE = [BaseCheck(), ConnectCheck(), GatherPackage(), InstallPackage(), ResultReport()] ISO_VERIFY_FLAG_STRING = "ISO 9660 CD-ROM filesystem data" @@ -42,12 +41,6 @@ if __name__ == '__main__': class_list=[CommandLine]) config_logging(CommandLine.silent) config_dict = read_yaml_file(CommandLine.yaml_path) - - if CommandLine.iso_path: - config_dict[constant.INSTRUCTION] = "deploy_iso" - check_iso_available(CommandLine.iso_path) - else: - config_dict[constant.INSTRUCTION] = "default" LOGGER.debug(f"-- config_dict: {config_dict}") pipe = PipeLine(config_dict) diff --git a/tools/install_dependency/src/exception/__init__.py b/tools/download_and_deploy/src/download/__init__.py similarity index 100% rename from tools/install_dependency/src/exception/__init__.py rename to tools/download_and_deploy/src/download/__init__.py diff --git a/tools/install_dependency/src/download/download_command_line.py b/tools/download_and_deploy/src/download/download_command_line.py similarity index 100% rename from tools/install_dependency/src/download/download_command_line.py rename to tools/download_and_deploy/src/download/download_command_line.py diff --git a/tools/install_dependency/src/download/download_config.py b/tools/download_and_deploy/src/download/download_config.py similarity index 87% rename from tools/install_dependency/src/download/download_config.py rename to tools/download_and_deploy/src/download/download_config.py index 2d3841e4c9defc80c39a33c569b2fd5d42542e03..207a83089354253d8af46b38b812b890b8c1b5d4 100644 --- a/tools/install_dependency/src/download/download_config.py +++ b/tools/download_and_deploy/src/download/download_config.py @@ -1,50 +1,52 @@ BiShengCompiler = { "component_name": "BiShengCompiler", "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_compiler/BiShengCompiler-3.2.0-aarch64-linux.tar.gz", - "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_compiler/BiShengCompiler-3.2.0-aarch64-linux.tar.gz.sha256", + "file_size": "1051195289", } GCCforOpenEuler = { "component_name": "GCCforOpenEuler", "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/kunpeng_gcc/gcc-10.3.1-2023.12-aarch64-linux.tar.gz", - "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/kunpeng_gcc/gcc-10.3.1-2023.12-aarch64-linux.tar.gz.sha256", + "file_size": "274901693", } BiShengJDK8 = { "component_name": "BiShengJDK8", "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-8u402-linux-aarch64.tar.gz", - "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-8u402-linux-aarch64.tar.gz.sha256", + "file_size": "117055434", } BiShengJDK17 = { "component_name": "BiShengJDK17", "file": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-17.0.10-linux-aarch64.tar.gz", - "sha256": "https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_jdk/bisheng-jdk-17.0.10-linux-aarch64.tar.gz.sha256", + "file_size": "196772672", } - -LkpTests = { - "component_name": "LkpTests", - "file": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/lkp-tests.tar.gz", - "GemDependency": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/gem_dependencies.zip", - "CompatibilityTesting": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/compatibility_testing.tar.gz", +CompatibilityTesting = { + "component_name": "CompatibilityTesting", + "file": "https://gitee.com/bdgl/devkit-pipeline/releases/download/compatibility_testing-v1.2/compatibility_testing.tar.gz", + "file_size": "76451308", } -DevkitTester = { +DevKitTester = { "component_name": "DevKitTester", - "file": "https://gitee.com/zoupp/devkit_mirrors/raw/master/document/devkit_tester/package/devkit_tester.tar.gz", + "file": "https://gitee.com/bdgl/devkit-pipeline/releases/download/compatibility_testing-v1.2/devkit_tester.tar.gz", + "file_size": "33426960", } A_FOT = { "component_name": "A-FOT", "file": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v0.2/a-fot.tar.gz", + "file_size": "15740", } DevKitWeb = { "component_name": "DevKitWeb", "file": "https://mirrors.huaweicloud.com/kunpeng/archive/DevKit/Packages/Kunpeng_DevKit/DevKit-All-24.0.RC1-Linux-Kunpeng.tar.gz", + "file_size": "1013920336", } DevKitCLI = { "component_name": "DevKitCLI", "file": "https://mirrors.huaweicloud.com/kunpeng/archive/DevKit/Packages/Kunpeng_DevKit/DevKit-CLI-24.0.RC1-Linux-Kunpeng.tar.gz", + "file_size": "413491454", } diff --git a/tools/install_dependency/src/download/download_utils.py b/tools/download_and_deploy/src/download/download_utils.py similarity index 33% rename from tools/install_dependency/src/download/download_utils.py rename to tools/download_and_deploy/src/download/download_utils.py index 14c55de742fef6e7af5e374306cbd471967e3206..c3f4bf7db20db06abadf14a0a420c0d07bf34313 100644 --- a/tools/install_dependency/src/download/download_utils.py +++ b/tools/download_and_deploy/src/download/download_utils.py @@ -14,107 +14,21 @@ warnings.filterwarnings("ignore", message='Unverified HTTPS request') component_collection_map = { - "BiShengCompiler": { - "download file": { - URL: f"{download_config.BiShengCompiler.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengCompiler.get(FILE).split('/')[-1])}", - FILE_SIZE: "1051195289", - }, - "download sha256": { - URL: f"{download_config.BiShengCompiler.get(SHA256)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengCompiler.get(SHA256).split('/')[-1])}", - FILE_SIZE: "107", - }, - }, - "GCCforOpenEuler": { - "download file": { - URL: f"{download_config.GCCforOpenEuler.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.GCCforOpenEuler.get(FILE).split('/')[-1])}", - FILE_SIZE: "274901693", - }, - "download sha256": { - URL: f"{download_config.GCCforOpenEuler.get(SHA256)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.GCCforOpenEuler.get(SHA256).split('/')[-1])}", - FILE_SIZE: "106", - }, - }, - "BiShengJDK8": { - "download file": { - URL: f"{download_config.BiShengJDK8.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK8.get(FILE).split('/')[-1])}", - FILE_SIZE: "117055434", - }, - "download sha256": { - URL: f"{download_config.BiShengJDK8.get(SHA256)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK8.get(SHA256).split('/')[-1])}", - FILE_SIZE: "105", - }, - }, - "BiShengJDK17": { - "download file": { - URL: f"{download_config.BiShengJDK17.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK17.get(FILE).split('/')[-1])}", - FILE_SIZE: "196772672", - }, - "download sha256": { - URL: f"{download_config.BiShengJDK17.get(SHA256)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.BiShengJDK17.get(SHA256).split('/')[-1])}", - FILE_SIZE: "107", - }, - }, - - "LkpTests": { - "download file": { - URL: f"{download_config.LkpTests.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get(FILE).split('/')[-1])}", - FILE_SIZE: "29333270", - }, - "GemDependency": { - URL: f"{download_config.LkpTests.get('GemDependency')}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get('GemDependency').split('/')[-1])}", - FILE_SIZE: "4206309", - }, - }, - - "CompatibilityTesting": { - "download file":{ - URL: f"{download_config.LkpTests.get('CompatibilityTesting')}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.LkpTests.get('CompatibilityTesting').split('/')[-1])}", - FILE_SIZE: "76645477", - } - }, - - "DevKitTester": { - "download file": { - URL: f"{download_config.DevkitTester.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.DevkitTester.get(FILE).split('/')[-1])}", - FILE_SIZE: "13349798", - } - }, - - "A-FOT": { - "download file": { - URL: f"{download_config.A_FOT.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.A_FOT.get(FILE).split('/')[-1])}", - FILE_SIZE: "15740", - } - }, - - "DevKitWeb": { - "download file": { - URL: f"{download_config.DevKitWeb.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.DevKitWeb.get(FILE).split('/')[-1])}", - FILE_SIZE: "1013920336", - }, - }, - - "DevKitCLI": { - "download file": { - URL: f"{download_config.DevKitCLI.get(FILE)}", - SAVE_PATH: f"{os.path.join(DEFAULT_PATH, download_config.DevKitCLI.get(FILE).split('/')[-1])}", - FILE_SIZE: "413491454", - } - }, + component.get("component_name"): { + URL: f"{component.get(FILE)}", + SAVE_PATH: f"{os.path.join(DEFAULT_PATH, component.get(FILE).split('/')[-1])}", + FILE_SIZE: f"{component.get(FILE_SIZE)}", + } for component in ( + download_config.BiShengCompiler, + download_config.GCCforOpenEuler, + download_config.BiShengJDK8, + download_config.BiShengJDK17, + download_config.CompatibilityTesting, + download_config.DevKitTester, + download_config.DevKitWeb, + download_config.DevKitCLI, + download_config.A_FOT + ) } @@ -129,29 +43,17 @@ def download_dependence(component_list): for component_name in component_collection_map: if component_name not in component_list: continue - shell_dict = component_collection_map.get(component_name) - ret = ret and download_dependence_handler(shell_dict) - return ret - - -def download_dependence_handler(shell_dict): - ret = True - for shell_cmd in shell_dict: - try: - ret = ret and download_dependence_file(shell_cmd, shell_dict) - except Exception as e: - ret = False + url_and_save_path = component_collection_map.get(component_name) + ret = ret and download_dependence_file(url_and_save_path) return ret -def download_dependence_file(shell_cmd, shell_dict): +def download_dependence_file(url_and_save_path): ret = True - url_and_save_path = shell_dict.get(shell_cmd) url_ = url_and_save_path.get("url") save_path = url_and_save_path.get("save_path") file_size = url_and_save_path.get("file_size") try: - print(f"Downloading from {url_}") download(url_, save_path, file_size) except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, @@ -167,7 +69,6 @@ def download_dependence_file(shell_cmd, shell_dict): f"\nPlease visit following url and download dependencies to default directory." f"\n\t{url_}" ) - raise OSError(f"download error occurs: {str(e)}") if not os.path.isfile(save_path) or not str(os.path.getsize(save_path)) == file_size: print(f"[ERROR] Download dependencies failed. " @@ -181,6 +82,7 @@ def download_dependence_file(shell_cmd, shell_dict): def download(url, save_path, file_size): if os.path.exists(save_path) and os.path.isfile(save_path) and str(os.path.getsize(save_path)) == file_size: return + print(f"Downloading from {url}") if platform.system() == "Windows": wget.download(url, save_path) print() diff --git a/tools/install_dependency/src/download_main.py b/tools/download_and_deploy/src/download_main.py similarity index 96% rename from tools/install_dependency/src/download_main.py rename to tools/download_and_deploy/src/download_main.py index 37034bb7532638b54183584068529acee9d76ae0..d577f786e2d8a86eddd6fa37e3d8fd7398e398c6 100644 --- a/tools/install_dependency/src/download_main.py +++ b/tools/download_and_deploy/src/download_main.py @@ -12,7 +12,7 @@ from handler.gather_package import GatherPackage from handler.compress_dep import CompressDep from utils import read_yaml_file -from constant import URL, SAVE_PATH, FILE, SHA256, FILE_SIZE, INSTRUCTION +from constant import URL, SAVE_PATH, FILE, SHA256, FILE_SIZE PIPELINE = [BaseCheck(), GatherPackage(), CompressDep()] @@ -83,7 +83,7 @@ if __name__ == '__main__': print("Download iso failed.") sys.exit(0) config_dict = read_yaml_file(CommandLine.yaml_path) - config_dict[INSTRUCTION] = "default" + pipe = PipeLine(config_dict) pipe.add_tail(*PIPELINE) pipe.start() diff --git a/tools/install_dependency/src/handler/__init__.py b/tools/download_and_deploy/src/exception/__init__.py similarity index 100% rename from tools/install_dependency/src/handler/__init__.py rename to tools/download_and_deploy/src/exception/__init__.py diff --git a/tools/install_dependency/src/exception/connect_exception.py b/tools/download_and_deploy/src/exception/connect_exception.py similarity index 100% rename from tools/install_dependency/src/exception/connect_exception.py rename to tools/download_and_deploy/src/exception/connect_exception.py diff --git a/tools/download_and_deploy/src/handler/__init__.py b/tools/download_and_deploy/src/handler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/install_dependency/src/handler/base_yaml_check.py b/tools/download_and_deploy/src/handler/base_yaml_check.py similarity index 83% rename from tools/install_dependency/src/handler/base_yaml_check.py rename to tools/download_and_deploy/src/handler/base_yaml_check.py index 3d2e89f041af5dd3414572e1c094f159eddb1922..cb75c11bffe5910b0d5ec09a10e44ea9ab8a9c20 100644 --- a/tools/install_dependency/src/handler/base_yaml_check.py +++ b/tools/download_and_deploy/src/handler/base_yaml_check.py @@ -5,12 +5,14 @@ from handler.handler_and_node import Handler from constant import ROLE_LIST -LOGGER = logging.getLogger("install_dependency") -MIN_SET = (constant.USER, constant.PKEY, constant.INSTRUCTION) +LOGGER = logging.getLogger("deploy_tool") +MIN_SET = (constant.USER, constant.PKEY) MAX_SET = (constant.USER, constant.PKEY, constant.PASSWORD, - constant.SCANNER, constant.JAVA_BUILDER_JDK8, constant.JAVA_BUILDER_JDK17, - constant.C_BUIDLER, constant.C_BUIDLER_GCC, constant.C_BUIDLER_BISHENG_COMPILER, constant.EXECUTOR, - constant.DEVKIT, constant.INSTRUCTION) + constant.SCANNER, + constant.C_BUILDER, constant.C_BUILDER_GCC, constant.C_BUILDER_BISHENG_COMPILER, + constant.JAVA_BUILDER, constant.JAVA_BUILDER_JDK8, constant.JAVA_BUILDER_JDK17, + constant.EXECUTOR, constant.COMPATIBILITY, constant.TESTER, + constant.DEVKIT, constant.CLAMAV) class BaseCheck(Handler): diff --git a/tools/install_dependency/src/handler/compress_dep.py b/tools/download_and_deploy/src/handler/compress_dep.py similarity index 100% rename from tools/install_dependency/src/handler/compress_dep.py rename to tools/download_and_deploy/src/handler/compress_dep.py diff --git a/tools/download_and_deploy/src/handler/connect_check.py b/tools/download_and_deploy/src/handler/connect_check.py new file mode 100644 index 0000000000000000000000000000000000000000..018d7589fa9d4f7a64471c9aa5c41bed09517c3d --- /dev/null +++ b/tools/download_and_deploy/src/handler/connect_check.py @@ -0,0 +1,52 @@ +import logging +import socket + +import constant +from handler.handler_and_node import Handler +from machine.machine import Machine +from exception.connect_exception import ConnectException +from constant import ROLE_COMPONENT, ROLE_LIST + +LOGGER = logging.getLogger("deploy_tool") + + +class ConnectCheck(Handler): + def handle(self, data) -> bool: + LOGGER.debug("ConnectCheck start!") + data[constant.MACHINE] = dict() + ret = True + for role in (ROLE_LIST & data.keys()): + ret = ret and ConnectCheck.machine_connect_check(data, role) + return ret + + @staticmethod + def machine_connect_check(data, role): + machine_list = data.get(role) + for ip in machine_list: + try: + machine_instance = data[constant.MACHINE].get(ip, None) + if not machine_instance: + machine_instance = Machine(ip, data[constant.USER], data[constant.PKEY], + data.get(constant.PASSWORD, None)) + + machine_instance.add_component(ROLE_COMPONENT[role]) + data[constant.MACHINE][ip] = machine_instance + except ConnectException: + LOGGER.error(f"-- [error] Connect {ip} failed. Please check.") + del data[constant.MACHINE] + return False + except Exception as e: + LOGGER.error(f"-- [error] Connect {ip} failed. Because of {str(e)}") + del data[constant.MACHINE] + return False + return True + + @staticmethod + def get_local_ip(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + sock.connect(("8.8.8.8", 80)) + ip = sock.getsockname()[0] + finally: + sock.close() + return ip if ip else "127.0.0.1" diff --git a/tools/install_dependency/src/handler/gather_package.py b/tools/download_and_deploy/src/handler/gather_package.py similarity index 97% rename from tools/install_dependency/src/handler/gather_package.py rename to tools/download_and_deploy/src/handler/gather_package.py index e03e674864e85930c5e21c0c576f65e057b656a0..87b5c16874ced00a239c850d84d938a2f667c71c 100644 --- a/tools/install_dependency/src/handler/gather_package.py +++ b/tools/download_and_deploy/src/handler/gather_package.py @@ -6,7 +6,7 @@ from download.download_utils import download_dependence from handler.handler_and_node import Handler from utils import generate_component_list -LOGGER = logging.getLogger("install_dependency") +LOGGER = logging.getLogger("deploy_tool") class GatherPackage(Handler): diff --git a/tools/install_dependency/src/handler/handler_and_node.py b/tools/download_and_deploy/src/handler/handler_and_node.py similarity index 100% rename from tools/install_dependency/src/handler/handler_and_node.py rename to tools/download_and_deploy/src/handler/handler_and_node.py diff --git a/tools/download_and_deploy/src/handler/install_package.py b/tools/download_and_deploy/src/handler/install_package.py new file mode 100644 index 0000000000000000000000000000000000000000..67aa584f313e42d09892f1ec838846fceeb917cc --- /dev/null +++ b/tools/download_and_deploy/src/handler/install_package.py @@ -0,0 +1,36 @@ +import logging +import multiprocessing + +import constant +from handler.handler_and_node import Handler + +LOGGER = logging.getLogger("deploy_tool") + + +class InstallPackage(Handler): + + def handle(self, data): + LOGGER.debug("Install Package start!") + + queue = multiprocessing.Queue() + jobs = [] + for _, machine in data[constant.MACHINE].items(): + process = multiprocessing.Process(target=process_work, args=(queue, machine,)) + jobs.append(process) + process.start() + for job in jobs: + job.join() + + while not queue.empty(): + modified_machine_instance = queue.get() + machine_instance = data[constant.MACHINE].get(modified_machine_instance.ip) + machine_instance.component_dict = modified_machine_instance.component_dict + return True + + +def process_work(queue, machine): + try: + machine.install_components() + queue.put(machine) + except (OSError, IOError) as e: + LOGGER.error(f"Remote machine {machine.ip} occur Error: {str(e)}") diff --git a/tools/install_dependency/src/handler/pipeline.py b/tools/download_and_deploy/src/handler/pipeline.py similarity index 91% rename from tools/install_dependency/src/handler/pipeline.py rename to tools/download_and_deploy/src/handler/pipeline.py index dbeaca6ed6594f8c5bf4532343dcae2dd1be7962..46278af8afef0a403c3c6f72eb8c8bb13ee3dc95 100644 --- a/tools/install_dependency/src/handler/pipeline.py +++ b/tools/download_and_deploy/src/handler/pipeline.py @@ -2,8 +2,6 @@ import logging from handler.handler_and_node import Node -LOGGER = logging.getLogger("install_dependency") - class PipeLine: """维护一个链表""" diff --git a/tools/download_and_deploy/src/handler/result_report.py b/tools/download_and_deploy/src/handler/result_report.py new file mode 100644 index 0000000000000000000000000000000000000000..de4e51fd842f6125031db81af43a6a4971dc3701 --- /dev/null +++ b/tools/download_and_deploy/src/handler/result_report.py @@ -0,0 +1,32 @@ +import logging + +import constant +from handler.handler_and_node import Handler +from machine.machine import Machine + +LOGGER = logging.getLogger("deploy_tool") + + +class ResultReport(Handler): + def handle(self, data) -> bool: + LOGGER.debug("Result report!") + machine_dict = data[constant.MACHINE] + + for ip in machine_dict: + machine_instance = machine_dict.get(ip) + ResultReport.report_machine_deploy_result(machine_instance) + return True + + @staticmethod + def report_machine_deploy_result(machine_instance): + report_str = f"Deploy result follow. \nRemote machine {machine_instance.ip} deploy result: \n" + component_dict = machine_instance.component_dict + + for component_name in component_dict: + deploy_result = component_dict[component_name] + if deploy_result: + report_str += f"\t{component_name} {deploy_result}\n" + else: + report_str += f"\t{component_name} install failed.\n" + LOGGER.info(f"{report_str}") + diff --git a/tools/install_dependency/src/log.py b/tools/download_and_deploy/src/log.py similarity index 89% rename from tools/install_dependency/src/log.py rename to tools/download_and_deploy/src/log.py index 64af387fbca6db2ca018b08c64c91469481571b7..40e6268df100e28e1d1027cb41394c137e229ab4 100644 --- a/tools/install_dependency/src/log.py +++ b/tools/download_and_deploy/src/log.py @@ -3,7 +3,7 @@ import sys def config_logging(silent=False): - logger = logging.getLogger("install_dependency") + logger = logging.getLogger("deploy_tool") logger.setLevel(logging.DEBUG) formatter = logging.Formatter( diff --git a/tools/download_and_deploy/src/machine/__init__.py b/tools/download_and_deploy/src/machine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/install_dependency/src/deploy/deploy_help.py b/tools/download_and_deploy/src/machine/machine.py similarity index 33% rename from tools/install_dependency/src/deploy/deploy_help.py rename to tools/download_and_deploy/src/machine/machine.py index 69953a6a1330c0951ccbddd1487a495eeb57df66..4dabeff0c04dae8d4072bcc99a34ba1a241cb726 100644 --- a/tools/install_dependency/src/deploy/deploy_help.py +++ b/tools/download_and_deploy/src/machine/machine.py @@ -5,26 +5,46 @@ import socket import paramiko import timeout_decorator +import constant +from component_install.deploy_base import DeployBase +from component_install.gcc_deploy import GccDeploy +from component_install.bisheng_compiler_deploy import BiShengCompilerDeploy +from component_install.bisheng_jdk8_deploy import BiShengJDK8Deploy +from component_install.bisheng_jdk17_deploy import BiShengJDK17Deploy +from component_install.compatibility_test_deploy import CompatibilityDeploy +from component_install.devkitweb_deploy import DevkitWebDeploy +from component_install.devkitcli_deploy import DevkitCLIDeploy +from component_install.devkittester_deploy import DevkitTesterDeploy +from component_install.switch_deploy import SwitchDeploy +from component_install.a_fot_deploy import AFotDeploy +from component_install.clamav_deploy import ClamAVDeploy + from exception.connect_exception import (CreatePkeyFailedException, ConnectRemoteException, NotMatchedMachineTypeException) -from utils import validate_path +from utils import validate_path, CHECK_TAR_AVAILABLE_CMD, MKDIR_TMP_DEVKITDEPENDENCIES_CMD + +LOGGER = logging.getLogger("deploy_tool") -class DeployHelp: - def __init__(self, ip, user, pkey, password, logger): + +class Machine: + def __init__(self, ip, user, pkey, password=None): self.ip = ip self.user = user self.pkey = pkey self.password = password - self.LOGGER = logger + self.check_is_aarch64() + self.component_dict = dict() - pass + def add_component(self, components): + for component in components: + self.component_dict[component] = [] def check_is_aarch64(self): machine_type = self.get_machine_type() - self.LOGGER.info(f"{self.ip} machine type: {machine_type}") + LOGGER.info(f"{self.ip} machine type: {machine_type}") if machine_type != "aarch64": - self.LOGGER.error(f"Machine type of {self.ip} is {machine_type}, not aarch64. Please replace this machine.") + LOGGER.error(f"Machine type of {self.ip} is {machine_type}, not aarch64. Please replace this machine.") raise NotMatchedMachineTypeException() def get_machine_type(self): @@ -32,7 +52,7 @@ class DeployHelp: ssh_client = self.ssh_client() stdin, stdout, stderr = ssh_client.exec_command("uname -m", timeout=10) except (paramiko.ssh_exception.SSHException, socket.timeout) as e: - self.LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") + LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") raise ConnectRemoteException() stdout_output = stdout.read().decode().strip() ssh_client.close() @@ -40,46 +60,85 @@ class DeployHelp: def ssh_client(self): ssh = paramiko.SSHClient() - ssh._transport = self.transport_connect() + ssh._transport = self.transport_connect(self.user, self.pkey, self.password) return ssh def sftp_client(self): - sftp = paramiko.SFTPClient.from_transport(self.transport_connect()) + sftp = paramiko.SFTPClient.from_transport(self.transport_connect(self.user, self.pkey, self.password)) return sftp - def transport_connect(self): - if not validate_path(self.pkey) or not os.path.isfile(self.pkey): - self.LOGGER.error("Yaml file content not correct. Given pkey not exists.") + def transport_connect(self, user, pkey_path, password=None): + if not validate_path(pkey_path) or not os.path.isfile(pkey_path): + LOGGER.error("Yaml file content not correct. Given pkey not exists.") raise ConnectRemoteException() try: # 指定本地的RSA私钥文件。如果建立密钥对时设置的有密码，password为设定的密码，如无不用指定password参数 - pkey = paramiko.RSAKey.from_private_key_file(self.pkey, password=self.password) + pkey = paramiko.RSAKey.from_private_key_file(pkey_path, password=password) except (IOError,) as e: - self.LOGGER.error(f"Pkey file not exists. {str(e)}") + LOGGER.error(f"Pkey file not exists. {str(e)}") raise CreatePkeyFailedException() except (paramiko.ssh_exception.PasswordRequiredException, paramiko.ssh_exception.AuthenticationException) as e: - self.LOGGER.warning(f"Pkey password is required. {str(e)}") - password = input(f"Press Enter to input password of {self.pkey}: ") + LOGGER.warning(f"Pkey password is required. {str(e)}") + password = input(f"Press Enter to input password of {pkey_path}: ") self.password = password - return self.transport_connect() + return self.transport_connect(user, pkey_path, password) except (paramiko.ssh_exception.SSHException,) as e: - self.LOGGER.error(f"Connect remote {self.ip} failed because of wrong pkey. {str(e)}") + LOGGER.error(f"Connect remote {self.ip} failed because of wrong pkey. {str(e)}") raise CreatePkeyFailedException() try: - transport = self.transport_connect_with_timeout(pkey) + transport = self.transport_connect_with_timeout(user, pkey) except (paramiko.ssh_exception.AuthenticationException, paramiko.ssh_exception.SSHException, timeout_decorator.TimeoutError, socket.gaierror, socket.timeout, socket.error) as e: - self.LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") + LOGGER.error(f"Connect remote {self.ip} failed. {str(e)}") raise ConnectRemoteException() return transport @timeout_decorator.timeout(10) - def transport_connect_with_timeout(self, pkey): + def transport_connect_with_timeout(self, user, pkey): transport = paramiko.Transport((self.ip, 22)) - transport.connect(username=self.user, pkey=pkey) + transport.connect(username=user, pkey=pkey) return transport + + def install_components(self): + ssh_client = self.ssh_client() + DeployBase._remote_exec_command(self.ip, ssh_client, CHECK_TAR_AVAILABLE_CMD) + DeployBase._remote_exec_command(self.ip, ssh_client, MKDIR_TMP_DEVKITDEPENDENCIES_CMD) + + for component in self.component_dict: + self.install_component(component) + DeployBase._clear_tmp_file_at_remote_machine( + self.ip, ssh_client, [os.path.join("/tmp/", constant.DEPENDENCY_DIR)]) + + def install_component(self, component_name): + ssh_client = self.ssh_client() + sftp_client = self.sftp_client() + try: + self.install_component_handler(component_name, sftp_client, ssh_client) + except (timeout_decorator.TimeoutError, FileNotFoundError, PermissionError, NotADirectoryError, + OSError, IOError) as e: + LOGGER.error(f"Remote machine {self.ip} install {component_name} occur Error: {str(e)}") + self.component_dict[component_name] = f"install failed. Error: {str(e)}" + finally: + ssh_client.close() + sftp_client.close() + + def install_component_handler(self, component_name, sftp_client, ssh_client): + component_name_to_func_dict = { + "GCCforOpenEuler": GccDeploy.install_handle, + "BiShengCompiler": BiShengCompilerDeploy.install_handle, + "BiShengJDK17": BiShengJDK17Deploy.install_handle, + "BiShengJDK8": BiShengJDK8Deploy.install_handle, + "CompatibilityTesting": CompatibilityDeploy.install_handle, + "DevKitWeb": DevkitWebDeploy.install_handle, + "DevKitCLI": DevkitCLIDeploy.install_handle, + "DevKitTester": DevkitTesterDeploy.install_handle, + "NonInvasiveSwitching": SwitchDeploy.install_handle, + "A-FOT": AFotDeploy.install_handle, + "ClamAV": ClamAVDeploy.install_handle, + } + return component_name_to_func_dict.get(component_name)(self, sftp_client, ssh_client) diff --git a/tools/install_dependency/src/utils.py b/tools/download_and_deploy/src/utils.py similarity index 60% rename from tools/install_dependency/src/utils.py rename to tools/download_and_deploy/src/utils.py index 285484042639eac8087c51da6491ee41eb49a4cc..cf2e5c7f3a12809e487bb1407bb3465683f92154 100644 --- a/tools/install_dependency/src/utils.py +++ b/tools/download_and_deploy/src/utils.py @@ -1,38 +1,39 @@ import os import sys import yaml -import paramiko -import socket -import timeout_decorator - +import logging from constant import ROLE_LIST, ROLE_COMPONENT CHECK_TAR_AVAILABLE_CMD = "which tar" CHECK_PERF_AVAILABLE_CMD = "which perf" MKDIR_TMP_DEVKITDEPENDENCIES_CMD = "mkdir -p /tmp/devkitdependencies" -YUM_INSTALL_LKP_DEPENDENCIES_CMD = "sudo yum install -y make git wget rubygems" CHECK_HOME_SPACE_SUFFICIENT_FOR_MIRROR = "[[ $(df -m /home | awk 'NR==2' | awk '{print $4}') -gt 17*1024 ]]" CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE = "[[ $(df -m /tmp | awk 'NR==2' | awk '{print $4}') -gt 1024 ]]" +CHECK_OPT_SPACE_SUFFICIENT_FOR_PACKAGE = "[[ $(df -m /opt | awk 'NR==2' | awk '{print $4}') -gt 2048 ]]" +CHECK_OPT_WRITE_PERMISSION = "[ -w /opt ]" +CHECK_SUDO_PERMISSION = "sudo -v &>/dev/null" CHECK_MIRROR_INSTALL_STATUS = "test -d /etc/yum.repos.d/yum.repos.backup" -SHELL_FILE_LIST = ["install.sh", "check_install_result.sh"] -global_value = { - "DEPLOY_ISO": False, - "component_list": [], - "deploy_component": [], - "ip": {} -} PROMPT_MAP = { - CHECK_TAR_AVAILABLE_CMD: "'tar' command not available.", - CHECK_PERF_AVAILABLE_CMD: "'perf' command not available.", + CHECK_TAR_AVAILABLE_CMD: "'tar' command not available, \n" + " please execute 'yum install tar -y' in remote machine.", + CHECK_PERF_AVAILABLE_CMD: "'perf' command not available, \n" + " please execute 'yum install perf -y' in remote machine.", MKDIR_TMP_DEVKITDEPENDENCIES_CMD: "Directory /tmp/devkitdependencies not exists.", - YUM_INSTALL_LKP_DEPENDENCIES_CMD: "Yum install lkp dependencies failed.", CHECK_HOME_SPACE_SUFFICIENT_FOR_MIRROR: "Machine /home space not sufficient for mirror.", CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE: "Machine /tmp space not sufficient for package.", + CHECK_OPT_SPACE_SUFFICIENT_FOR_PACKAGE: "Machine /opt space not sufficient for package.", + CHECK_OPT_WRITE_PERMISSION: "Machine /opt write permission not sufficient for current user, \n" + " please consider using root to install this component or change /opt permission to 757.", + CHECK_SUDO_PERMISSION: "Machine sudo permission not configure for current user, please consider \n" + " 1. add sudo permission to current user.\n" + " 2. add sudo password-free to current user.\n", CHECK_MIRROR_INSTALL_STATUS: "Mirror mount status not correct.", } +LOGGER = logging.getLogger("deploy_tool") + def validate_path(path: str) -> bool: return path.startswith('/') and path.find('../') == -1 and path.find('./') == -1 @@ -69,18 +70,3 @@ def generate_component_list(yaml_dict): continue component_list.extend(ROLE_COMPONENT[role]) return list(set(component_list)) - - -@timeout_decorator.timeout(100) -def remote_exec_command(cmd, ssh_client, LOGGER, ip): - try: - stdin, stdout, stderr = ssh_client.exec_command(cmd, timeout=90) - except (paramiko.ssh_exception.SSHException, socket.timeout) as e: - LOGGER.error(f"Remote machine {ip} exec '{cmd}' failed Please run this command in this machine.") - raise OSError(PROMPT_MAP.get(cmd, f"Remote machine {ip} exec '{cmd}' failed.")) - exit_status = stdout.channel.recv_exit_status() - if exit_status == 0: - LOGGER.debug(f"Remote machine {ip} exec '{cmd}' success.") - else: - LOGGER.error(f"Remote machine {ip} exec '{cmd}' failed. Please run this command in this machine.") - raise OSError(PROMPT_MAP.get(cmd, f"Remote machine {ip} exec '{cmd}' failed.")) diff --git a/tools/install_dependency/src/deploy/a_fot_deploy.py b/tools/install_dependency/src/deploy/a_fot_deploy.py deleted file mode 100644 index 08324236d8cf666b0b7d70b43f26089f18d9f63f..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/a_fot_deploy.py +++ /dev/null @@ -1,23 +0,0 @@ -import constant -import os -from deploy.deploy_base import DeployBase -from utils import (CHECK_PERF_AVAILABLE_CMD, remote_exec_command, MKDIR_TMP_DEVKITDEPENDENCIES_CMD) - - -class AFotDeploy(DeployBase): - def __init__(self, component, - ip, user, pkey, password): - super(AFotDeploy, self).__init__(component, - ip, user, pkey, password, hook_before_upload_fn=self.hook_before_upload_fn, hook_after_install_fn=None) - self.upload_func = False - - def hook_before_upload_fn(self): - remote_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD, self.ssh_client, self.LOGGER, self.ip) - remote_exec_command(CHECK_PERF_AVAILABLE_CMD, self.ssh_client, self.LOGGER, self.ip) - saved_path = os.path.join(constant.DEFAULT_PATH, "a-fot.tar.gz") - remote_file = os.path.abspath(os.path.join('/tmp', saved_path)) - self.LOGGER.debug(f"Transport local_file: {saved_path} to remote machine {self.ip} " - f"remote_file: {remote_file}") - self.sftp_client.put(localpath=f"{saved_path}", remotepath=f"{remote_file}") - - diff --git a/tools/install_dependency/src/deploy/bisheng_compiler_deploy.py b/tools/install_dependency/src/deploy/bisheng_compiler_deploy.py deleted file mode 100644 index 865a4b9dceea2543fe74468b5b756f256892b03c..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/bisheng_compiler_deploy.py +++ /dev/null @@ -1,6 +0,0 @@ - -from deploy.deploy_base import DeployBase - - -class BiShengCompilerDeploy(DeployBase): - pass diff --git a/tools/install_dependency/src/deploy/bisheng_jdk17_deploy.py b/tools/install_dependency/src/deploy/bisheng_jdk17_deploy.py deleted file mode 100644 index 6bbf791a0a19135a3615915bed1b113ef196810b..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/bisheng_jdk17_deploy.py +++ /dev/null @@ -1,6 +0,0 @@ - -from deploy.deploy_base import DeployBase - - -class BiShengJDK17Deploy(DeployBase): - pass diff --git a/tools/install_dependency/src/deploy/bisheng_jdk8_deploy.py b/tools/install_dependency/src/deploy/bisheng_jdk8_deploy.py deleted file mode 100644 index a6acb2e4a4a139d3e93f42af423d51ca82563584..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/bisheng_jdk8_deploy.py +++ /dev/null @@ -1,6 +0,0 @@ - -from deploy.deploy_base import DeployBase - - -class BiShengJDK8Deploy(DeployBase): - pass diff --git a/tools/install_dependency/src/deploy/compatibility_test_deploy.py b/tools/install_dependency/src/deploy/compatibility_test_deploy.py deleted file mode 100644 index e5bc0f46108a90d9ebf4ab6639fb2e14ef6acb95..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/compatibility_test_deploy.py +++ /dev/null @@ -1,10 +0,0 @@ - -from deploy.deploy_base import DeployBase - - -class CompatibilityDeploy(DeployBase): - def __init__(self, component, - ip, user, pkey, password): - super(CompatibilityDeploy, self).__init__(component, - ip, user, pkey, password, hook_before_upload_fn=None, hook_after_install_fn=None) - self.sudo = True diff --git a/tools/install_dependency/src/deploy/deploy_base.py b/tools/install_dependency/src/deploy/deploy_base.py deleted file mode 100644 index bfbfeba8110b640a5b28d8c6df423b54344551ae..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/deploy_base.py +++ /dev/null @@ -1,135 +0,0 @@ -import logging -import os - -import timeout_decorator - -import constant -from deploy.deploy_help import DeployHelp -from download.download_utils import component_collection_map -from utils import (base_path, MKDIR_TMP_DEVKITDEPENDENCIES_CMD, - CHECK_TAR_AVAILABLE_CMD, CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE, - CHECK_MIRROR_INSTALL_STATUS, remote_exec_command, SHELL_FILE_LIST, global_value) - - -class DeployBase: - def __init__(self, component, - ip, user, pkey, password, hook_before_upload_fn=None, hook_after_install_fn=None): - self.component = component - self.before_upload_fn = hook_before_upload_fn - self.after_install_fn = hook_after_install_fn - self.LOGGER = logging.getLogger("install_dependency") - self.ip = ip - self.user = user - self.pkey = pkey - self.upload_func = True - self.install_func = True - self.install_param = False - self.password = password - self.deploy_helper = DeployHelp(ip, user, pkey, password, self.LOGGER) - self.sftp_client = None - self.ssh_client = None - self.sudo = False - self.remote_file_list = [] - # 去掉这个，每个component直接以compoent去调用 - - def clear_tmp_file_at_remote_machine(self, remote_file_list): - self.LOGGER.debug(f"Clear tmp file at remote machine {self.ip}") - for remote_file in remote_file_list: - try: - remote_file = os.path.realpath(remote_file) - if not remote_file.startswith(os.path.join("/tmp", constant.DEPENDENCY_DIR)): - continue - self.LOGGER.debug(f"Delete tmp file at remote machine {self.ip}: {remote_file}") - self.ssh_client.exec_command(f"rm -fr {remote_file}") - except Exception as e: - self.LOGGER.debug(str(e)) - - def upload(self): - remote_exec_command(CHECK_TAR_AVAILABLE_CMD, self.ssh_client, self.LOGGER, self.ip) - remote_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD, self.ssh_client, self.LOGGER, self.ip) - remote_exec_command(CHECK_TMP_SPACE_SUFFICIENT_FOR_PACKAGE, self.ssh_client, self.LOGGER, self.ip) - # 上传 lkp-tests.tar.gz文件 - self.LOGGER.info(f"Install component in remote machine {self.ip}: {self.component}") - - shell_dict = component_collection_map.get(self.component) - for shell_cmd in shell_dict: - url_and_save_path = shell_dict.get(shell_cmd) - local_file = url_and_save_path.get("save_path") - remote_file = os.path.abspath(os.path.join('/tmp', constant.DEPENDENCY_DIR, local_file.split('/')[-1])) - self.LOGGER.debug(f"Transport local_file: {local_file} to remote machine {self.ip} " - f"remote_file: {remote_file}") - - self.remote_file_list.append(remote_file) - self.sftp_client.put(localpath=f"{local_file}", remotepath=f"{remote_file}") - - def transport_shell_file_and_execute(self, sh_file_local_path, sh_file_remote_path, - sh_cmd): - if not os.path.exists(sh_file_local_path): - self.LOGGER.error(f"{sh_file_local_path} not exists.") - raise FileNotFoundError(f"local file {sh_file_local_path} not exists.") - - self.LOGGER.debug(f"Transport local_file: {sh_file_local_path} to remote machine {self.ip} " - f"remote_file: {sh_file_remote_path}") - self.sftp_client.put(localpath=sh_file_local_path, remotepath=sh_file_remote_path) - - stdin, stdout, stderr = self.ssh_client.exec_command(sh_cmd) - output = stdout.read().decode().strip() - self.LOGGER.info(f"Remote machine {self.ip} '{sh_cmd}' output: {output}") - return output - - def install(self): - # 上传并执行 安装脚本, 校验安装结果脚本 - install_result = "" - for shell_file in SHELL_FILE_LIST: - sh_file_local_path = os.path.join(base_path("component"), self.component, shell_file) - sh_file_remote_path = os.path.join("/tmp/", constant.DEPENDENCY_DIR, self.component + shell_file) - if self.install_param: - if self.sudo: - sh_cmd = f"sudo bash {sh_file_remote_path} {sh_file_remote_path}" - else: - sh_cmd = f"bash {sh_file_remote_path} {sh_file_remote_path}" - else: - if self.sudo: - sh_cmd = f"sudo bash {sh_file_remote_path}" - else: - sh_cmd = f"bash {sh_file_remote_path}" - execute_output = ( - self.transport_shell_file_and_execute( - sh_file_local_path=sh_file_local_path, - sh_file_remote_path=sh_file_remote_path, - sh_cmd=sh_cmd - )) - self.remote_file_list.append(sh_file_remote_path) - if shell_file == SHELL_FILE_LIST[1]: - install_result = execute_output - - if install_result == "true": - self.LOGGER.info(f"Remote machine {self.ip} install {self.component} success.") - else: - self.LOGGER.error(f"Remote machine {self.ip} install {self.component} failed.") - - def clean(self): - # 清理tmp临时文件 - self.clear_tmp_file_at_remote_machine(self.remote_file_list) - - def install_component_handler(self): - try: - self.sftp_client = self.deploy_helper.sftp_client() - self.ssh_client = self.deploy_helper.ssh_client() - if self.before_upload_fn: - self.before_upload_fn() - if self.upload_func: - self.upload() - if self.install_func: - self.install() - if self.after_install_fn: - self.after_install_fn() - self.clean() - except timeout_decorator.TimeoutError as e: - self.LOGGER.error(f"Remote machine {self.ip} occur Error: Exec cmd {str(e)}") - except (FileNotFoundError, PermissionError, NotADirectoryError, OSError, IOError) as e: - self.LOGGER.error(f"Remote machine {self.ip} occur Error: {str(e)}") - finally: - self.clear_tmp_file_at_remote_machine([os.path.join("/tmp/", constant.DEPENDENCY_DIR)]) - self.ssh_client.close() - self.sftp_client.close() diff --git a/tools/install_dependency/src/deploy/devkit_distribute_deploy.py b/tools/install_dependency/src/deploy/devkit_distribute_deploy.py deleted file mode 100644 index 50a5a0284ab3b93fb7a5f68a102256397736da8a..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/devkit_distribute_deploy.py +++ /dev/null @@ -1,6 +0,0 @@ - -from deploy.deploy_base import DeployBase - - -class DevkitTesterDeploy(DeployBase): - pass \ No newline at end of file diff --git a/tools/install_dependency/src/deploy/devkitcli_deploy.py b/tools/install_dependency/src/deploy/devkitcli_deploy.py deleted file mode 100644 index 9c6aacf2c30991605e2c12d9d998c3d89f887cce..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/devkitcli_deploy.py +++ /dev/null @@ -1,10 +0,0 @@ - -from deploy.deploy_base import DeployBase - - -class DevkitCLIDeploy(DeployBase): - def __init__(self, component, - ip, user, pkey, password): - super(DevkitCLIDeploy, self).__init__(component, - ip, user, pkey, password, hook_before_upload_fn=None, hook_after_install_fn=None) - self.sudo = True diff --git a/tools/install_dependency/src/deploy/devkitweb_deploy.py b/tools/install_dependency/src/deploy/devkitweb_deploy.py deleted file mode 100644 index 46296b517acd9ed54b79033cf4c15f814504f034..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/devkitweb_deploy.py +++ /dev/null @@ -1,44 +0,0 @@ -# todo 这个原来的逻辑写的跟咱们通用的有点不一样要改 - -import os -import subprocess -from download.download_utils import component_collection_map -from deploy.deploy_base import DeployBase -from utils import base_path, MKDIR_TMP_DEVKITDEPENDENCIES_CMD, remote_exec_command - - -class DevkitWebDeploy(DeployBase): - def __init__(self, component, - ip, user, pkey, password): - super(DevkitWebDeploy, self).__init__(component, - ip, user, pkey, password, hook_before_upload_fn=self.hook_before_upload_fn, hook_after_install_fn=self.hook_after_install_fn) - self.install_func = False - self.upload_func = False - - def hook_before_upload_fn(self): - remote_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD, self.ssh_client, self.LOGGER, self.ip) - - self.LOGGER.info(f"Install component in remote machine {self.ip}: {self.component}") - shell_dict = component_collection_map.get(self.component) - - url_and_save_path = shell_dict.get("download file") - local_file = url_and_save_path.get("save_path") - remote_file = os.path.abspath(os.path.join('/opt', local_file.split('/')[-1])) - self.LOGGER.debug(f"Transport local_file: {local_file} to remote machine {self.ip} " - f"remote_file: {remote_file}") - self.remote_file_list.append(remote_file) - self.sftp_client.put(localpath=f"{local_file}", remotepath=f"{remote_file}") - - cmd = f"{os.path.join(base_path('component'), self.component, 'devkit_installer')} " \ - f"-i {self.ip} -u {self.user} -p {self.pkey} -paname {local_file.split('/')[-1]}" - self.LOGGER.debug(f"Executing command: {cmd}") - subprocess.run(cmd.split(' '), - capture_output=False, shell=False, stderr=subprocess.STDOUT) - - def hook_after_install_fn(self): - local_file = component_collection_map.get(self.component).get("download file").get("save_path") - cmd = f"{os.path.join(base_path('component'), self.component, 'devkit_installer')} " \ - f"-i {self.ip} -u {self.user} -p {self.pkey} -paname {local_file.split('/')[-1]}" - self.LOGGER.debug(f"Executing command: {cmd}") - subprocess.run(cmd.split(' '), - capture_output=False, shell=False, stderr=subprocess.STDOUT) diff --git a/tools/install_dependency/src/deploy/gcc_deploy.py b/tools/install_dependency/src/deploy/gcc_deploy.py deleted file mode 100644 index 06caba845442ef84a4fffe0d0922aaf603456594..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/gcc_deploy.py +++ /dev/null @@ -1,6 +0,0 @@ - -from deploy.deploy_base import DeployBase - - -class GccDeploy(DeployBase): - pass diff --git a/tools/install_dependency/src/deploy/lkptest_deploy.py b/tools/install_dependency/src/deploy/lkptest_deploy.py deleted file mode 100644 index 955fd3bd1a67eb9f5b47a97bd672b145cb65881c..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/lkptest_deploy.py +++ /dev/null @@ -1,14 +0,0 @@ - -from deploy.deploy_base import DeployBase -from utils import YUM_INSTALL_LKP_DEPENDENCIES_CMD, remote_exec_command - - -class LkpTestDeploy(DeployBase): - def __init__(self, component, - ip, user, pkey, password): - super(LkpTestDeploy, self).__init__(component, - ip, user, pkey, password, hook_before_upload_fn=self.hook_before_upload_fn, hook_after_install_fn=None) - self.sudo = True - - def hook_before_upload_fn(self): - remote_exec_command(YUM_INSTALL_LKP_DEPENDENCIES_CMD, self.ssh_client, self.LOGGER, self.ip) diff --git a/tools/install_dependency/src/deploy/mount_mirror_deploy.py b/tools/install_dependency/src/deploy/mount_mirror_deploy.py deleted file mode 100644 index fbb29480e3b391df6c3a2b4e45d5943ba5a840a7..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/mount_mirror_deploy.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -from deploy.deploy_base import DeployBase -from utils import CHECK_HOME_SPACE_SUFFICIENT_FOR_MIRROR, remote_exec_command, MKDIR_TMP_DEVKITDEPENDENCIES_CMD -from deploy.deploy_command_line import CommandLine - - -class MountISODeploy(DeployBase): - def __init__(self, component, - ip, user, pkey, password): - super(MountISODeploy, self).__init__(component, - ip, user, pkey, password, hook_before_upload_fn=self.hook_before_upload_fn, - hook_after_install_fn=None) - self.upload_func = False - self.install_param = True - - def hook_before_upload_fn(self): - remote_exec_command(CHECK_HOME_SPACE_SUFFICIENT_FOR_MIRROR, self.ssh_client, self.LOGGER, self.ip) - remote_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD, self.ssh_client, self.LOGGER, self.ip) - self.LOGGER.info(f"Deploy component in remote machine {self.ip}: {self.component}") - local_path = os.path.abspath(CommandLine.iso_path) - remote_path = os.path.join("/home", local_path.split('/')[-1]) - self.LOGGER.debug(f"Transport local_file: {local_path} to remote machine {self.ip} " - f"remote_file: {remote_path}") - self.sftp_client.put(localpath=local_path, remotepath=remote_path) diff --git a/tools/install_dependency/src/deploy/switch_deploy.py b/tools/install_dependency/src/deploy/switch_deploy.py deleted file mode 100644 index 0d119f2b4df9be332b52bf1458a9a3701c9af03a..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/switch_deploy.py +++ /dev/null @@ -1,15 +0,0 @@ -from deploy.deploy_base import DeployBase -from utils import remote_exec_command, MKDIR_TMP_DEVKITDEPENDENCIES_CMD - - -class SwitchDeploy(DeployBase): - def __init__(self, component, - ip, user, pkey, password): - super(SwitchDeploy, self).__init__(component, - ip, user, pkey, password, hook_before_upload_fn=self.hook_before_upload_fn, - hook_after_install_fn=None) - self.upload_func = False - - def hook_before_upload_fn(self): - remote_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD, self.ssh_client, self.LOGGER, self.ip) - diff --git a/tools/install_dependency/src/deploy/unmount_mirror_deploy.py b/tools/install_dependency/src/deploy/unmount_mirror_deploy.py deleted file mode 100644 index b1fd90911b13596efaf3d04e7d532132d19ff4c1..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/deploy/unmount_mirror_deploy.py +++ /dev/null @@ -1,37 +0,0 @@ -import os - -from deploy.deploy_base import DeployBase -from utils import CHECK_MIRROR_INSTALL_STATUS, base_path, remote_exec_command, MKDIR_TMP_DEVKITDEPENDENCIES_CMD -from deploy.deploy_command_line import CommandLine - - -class UnMountISODeploy(DeployBase): - def __init__(self, component, - ip, user, pkey, password): - super(UnMountISODeploy, self).__init__(component, - ip, user, pkey, password, - hook_before_upload_fn=self.hook_before_upload_fn, - hook_after_install_fn=self.hook_after_install_fn) - self.install_func = False - self.upload_func = False - - def hook_before_upload_fn(self): - remote_exec_command(CHECK_MIRROR_INSTALL_STATUS, self.ssh_client, self.LOGGER, self.ip) - self.component = self.component.replace("Un", "") - - def hook_after_install_fn(self): - remote_exec_command(MKDIR_TMP_DEVKITDEPENDENCIES_CMD, self.ssh_client, self.LOGGER, self.ip) - local_path = os.path.abspath(CommandLine.iso_path) - remote_path = os.path.join("/home", local_path.split('/')[-1]) - for shell_file in ["uninstall.sh"]: - sh_file_local_path = os.path.join(base_path("component"), self.component, shell_file) - sh_file_remote_path = os.path.join("/tmp/", self.component + shell_file) - sh_cmd = f"bash {sh_file_remote_path}" - execute_output = ( - self.transport_shell_file_and_execute( - sh_file_local_path=sh_file_local_path, - sh_file_remote_path=sh_file_remote_path, - sh_cmd=sh_cmd - )) - self.remote_file_list.append(sh_file_remote_path) - self.remote_file_list.append(remote_path) diff --git a/tools/install_dependency/src/handler/connect_check.py b/tools/install_dependency/src/handler/connect_check.py deleted file mode 100644 index e2602f9e7be1def1dc795cf2d9886c32cb668bba..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/handler/connect_check.py +++ /dev/null @@ -1,91 +0,0 @@ -import logging -import socket - -import constant -from constant import ROLE_COMPONENT, ROLE_LIST -from deploy.a_fot_deploy import AFotDeploy -from deploy.bisheng_compiler_deploy import BiShengCompilerDeploy -from deploy.bisheng_jdk17_deploy import BiShengJDK17Deploy -from deploy.bisheng_jdk8_deploy import BiShengJDK8Deploy -from deploy.compatibility_test_deploy import CompatibilityDeploy -from deploy.devkit_distribute_deploy import DevkitTesterDeploy -from deploy.devkitcli_deploy import DevkitCLIDeploy -from deploy.devkitweb_deploy import DevkitWebDeploy -from deploy.gcc_deploy import GccDeploy -from deploy.lkptest_deploy import LkpTestDeploy -from deploy.mount_mirror_deploy import MountISODeploy -from deploy.switch_deploy import SwitchDeploy -from deploy.unmount_mirror_deploy import UnMountISODeploy -from exception.connect_exception import ConnectException -from handler.handler_and_node import Handler -from utils import global_value - -LOGGER = logging.getLogger("install_dependency") -ROLE_MAP = { - "BiShengJDK17": BiShengJDK17Deploy, - "BiShengJDK8": BiShengJDK8Deploy, - "DevKitCLI": DevkitCLIDeploy, - "GCCforOpenEuler": GccDeploy, - "BiShengCompiler": BiShengCompilerDeploy, - "A-FOT": AFotDeploy, - "NonInvasiveSwitching": SwitchDeploy, - "LkpTests": LkpTestDeploy, - "CompatibilityTesting": CompatibilityDeploy, - "DevkitTester": DevkitTesterDeploy, - "DevKitWeb": DevkitWebDeploy, - "OpenEulerMirrorISO": MountISODeploy, - "UnOpenEulerMirrorISO": UnMountISODeploy -} - - -class ConnectCheck(Handler): - def handle(self, data) -> bool: - LOGGER.debug("ConnectCheck start!") - data[constant.MACHINE] = dict() - ret = True - for role in (ROLE_LIST & data.keys()): - ret = ret and ConnectCheck.deploy_role_check(data, role) - return ret - - @staticmethod - def deploy_role_check(data, role): - builder_list = data.get(role) - if data.get(constant.INSTRUCTION) == "deploy_iso": - global_value["DEPLOY_ISO"] = True - for ip in builder_list: - try: - for each in ROLE_COMPONENT[role]: - if ip in global_value["ip"].keys(): - global_value["ip"][ip]["component_set"].add(each) - else: - global_value["ip"][ip] = dict() - global_value["ip"][ip]["component_set"] = set() - global_value["ip"][ip]["component_set"].add(each) - global_value["ip"][ip]["component_list"] = list(global_value["ip"][ip]["component_set"]) - if global_value["DEPLOY_ISO"]: - global_value["ip"][ip]["component_list"].insert(0, "OpenEulerMirrorISO") - global_value["ip"][ip]["component_list"].append("UnOpenEulerMirrorISO") - global_value["ip"][ip]["deploy_component"] = [] - for each in global_value["ip"][ip]["component_list"]: - global_value["ip"][ip]["deploy_component"].append(ROLE_MAP.get(each)(each, ip, data[constant.USER], - data[constant.PKEY], - data.get(constant.PASSWORD))) - except ConnectException: - LOGGER.error(f"-- [error] Connect {ip} failed. Please check.") - del data[constant.MACHINE] - return False - except Exception as e: - LOGGER.error(f"-- [error] Connect {ip} failed. Because of {str(e)}") - del data[constant.MACHINE] - return False - return True - - @staticmethod - def get_local_ip(): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - sock.connect(("8.8.8.8", 80)) - ip = sock.getsockname()[0] - finally: - sock.close() - return ip if ip else "127.0.0.1" diff --git a/tools/install_dependency/src/handler/install_package.py b/tools/install_dependency/src/handler/install_package.py deleted file mode 100644 index c11a496643fbf06bd23c6d15fb5e3b8b3bf26d15..0000000000000000000000000000000000000000 --- a/tools/install_dependency/src/handler/install_package.py +++ /dev/null @@ -1,40 +0,0 @@ -import logging -import multiprocessing - -import constant -from utils import global_value -from handler.handler_and_node import Handler - -LOGGER = logging.getLogger("install_dependency") - - -class InstallPackage(Handler): - - def handle(self, data): - LOGGER.debug("Install Package start!") - jobs = [] - undeploy_iso = None - for each in global_value["ip"].keys(): - process = multiprocessing.Process(target=process_deploy_component, args=(each,)) - jobs.append(process) - process.start() - for job in jobs: - job.join() - return True - - -def process_deploy_component(ip): - try: - undeploy_iso = None - if global_value["DEPLOY_ISO"]: - global_value["ip"][ip]["deploy_component"][0].install_component_handler() - undeploy_iso = global_value["ip"][ip]["deploy_component"][-1] - del global_value["ip"][ip]["deploy_component"][0] - del global_value["ip"][ip]["deploy_component"][-1] - for every in global_value["ip"][ip]["deploy_component"]: - every.install_component_handler() - if global_value["DEPLOY_ISO"]: - undeploy_iso.install_component_handler() - except (OSError, IOError) as e: - LOGGER.error(f"Remote machine {ip} occur Error: {str(e)}") -