diff --git a/plugins/intelligent-parameter-tuning/README.md b/plugins/intelligent-parameter-tuning/README.md index 52a48f94dae5417018dcd5227eca1b7af1632411..83b1f67528c0632f4ca4c0f0c1e988128f05d567 100644 --- a/plugins/intelligent-parameter-tuning/README.md +++ b/plugins/intelligent-parameter-tuning/README.md @@ -58,14 +58,12 @@ ##### 注意事项 jdk版本为11+ -maven版本为3.6.3 datakit 所在服务器环境要求: 1. yum 命令能够正常使用 -2. python环境 python3.7+,能够正确执行python3 命令 -3. 每一个openGauss数据库集群第一次训练会校验python环境是否正常安装项目需要的依赖,如果返回python环境失败，或者hebo安装失败 -4. 假设datakit的安装目录为/usr/local/work,前往/usr/local/work/data/tuning/log/, -5. 查看训练日志文件 +2. python环境 python3.7+,能够正确执行python3,pip3 命令 +3. 如果是sysbench压测模式,会检测是否安装sysbench,没安装时,请安装sysbench,每一个openGauss数据库集群第一次训练会校验python环境是否正常安装项目需要的依赖,没有安装对应的依赖,执行下面命令: + 执行以下命令： * yum -y install postgresql-devel * yum install gcc libffi-devel python3-devel openssl-devel -y -* pip3 install -r requirements.txt, -* pip3 install -e .的执行详情 \ No newline at end of file +4. datakit平台,在服务器管理页面,配置被测数据库所在的服务器信息,安装数据库的用户信息,实例管理配置被测数据库的连接信息 +5. 被测数据库在测试过程中最好是空闲状态,不被其他服务所依赖使用 \ No newline at end of file diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/common/FixedTuning.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/common/FixedTuning.java index c16fc21706407db65910052e68789c07f080f1b2..05a6481afe854e8b1a00870298cde0bede1a9292 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/common/FixedTuning.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/common/FixedTuning.java @@ -43,6 +43,19 @@ public class FixedTuning { */ public static final String CHECK_PYTHON_VERSION = "python3 -c 'import sys; print(sys.version_info >= (3, 7, 0))'"; + /** + * CHECK_SYSBENCH + */ + public static final String CHECK_SYSBENCH = "command -v sysbench > /dev/null 2>&1 && echo True || echo False"; + + /** + * Check dependency items + */ + public static final String CHECK_RELY = "missing=\"\";rpm -q openssl-devel > /dev/null " + + "|| missing+=\"openssl-devel \"; " + + "rpm -q python3-devel > /dev/null || missing+=\"python3-devel \"; rpm -q libffi-devel > /dev/null" + + " || missing+=\"libffi-devel \"; [ -n \"$missing\" ] && echo \"请安装$missing\" || echo \"\""; + /** * BOOL_TRUE */ @@ -119,34 +132,30 @@ public class FixedTuning { public static final Integer UNZIP = 5; /** - * PYTHON_ENV_PROC + * initValue */ - public static final Integer PYTHON_ENV_PROC = 10; + public static final Integer INIT_VALUE = 0; /** - * PYTHON_ENV_PROC + * SEARCH_DATABASE */ - public static final Integer PYTHON_HEBO_PROC = 15; + public static final String SEARCH_DATABASE = "SELECT datname FROM pg_database"; /** - * PYTHON_STRATEGY_PROC + * SEARCH_RES */ - public static final Integer PYTHON_STRATEGY_PROC = 20; + public static final String SEARCH_RES = "datname"; - /** - * initValue - */ - public static final Integer INIT_VALUE = 0; /** * SEARCH_DATABASE */ - public static final String SEARCH_DATABASE = "SELECT datname FROM pg_database"; + public static final String SEARCH = "SELECT nspname FROM pg_namespace"; /** * SEARCH_RES */ - public static final String SEARCH_RES = "datname"; + public static final String RES = "nspname"; /** * EXECUTE_JOB @@ -226,40 +235,20 @@ public class FixedTuning { */ public static final String SYSBENCH_EXECUTE_PATH = "data"; - /** - * ENV_YUM - */ - public static final String ENV_YUM = "yum -y install postgresql-devel" - + "&& yum install gcc libffi-devel python3-devel openssl-devel -y" + DETERMINE; - - /** - * CREATA_SYSBENCH_ENV - */ - public static final String CREATA_SYSBENCH_ENV = "yum -y install make automake libtool pkgconfig libaio-devel " - + "&& yum -y install mariadb-devel openssl-devel " - + "&& ./autogen.sh " - + "&& ./configure --with-pgsql " - + "&& make -j && make install " + DETERMINE; - /** * PYTHON3_ENV */ public static final String PYTHON3_ENV = "pip3 install -r requirements.txt " + DETERMINE; - /** - * CHECK_SYSBENCH_VERSION - */ - public static final String CHECK_SYSBENCH_VERSION = "sysbench --version " + DETERMINE; - /** * JSON_HELPER_COMMAND */ - public static final String JSON_HELPER_COMMAND = "python3 SuperWG/DWG/jsonHelper/jsonHelper.py " + DETERMINE; + public static final String JSON_HELPER_COMMAND = "python3 jsonHelper.py " + DETERMINE; /** * WORKLOAD_PATH */ - public static final String WORKLOAD_COMMAND = "python3 SuperWG/DWG/src/WorkloadGenerator.py " + DETERMINE; + public static final String WORKLOAD_COMMAND = "python3 WorkloadGenerator.py " + DETERMINE; /** * LOAD_FILE_PATH @@ -314,17 +303,17 @@ public class FixedTuning { /** * APPLY_TO_DATABASE_COMMAND_OFFLINE */ - public static final String APPLY_TO_DATABASE_COMMAND_OFFLINE = "gs_guc set -c %s -D %s"; + public static final String APPLY_TO_DATABASE_COMMAND_OFFLINE = "gs_guc set -Z datanode -N all -I all -c %s"; /** * APPLY_TO_DATABASE_COMMAND_ONLINE */ - public static final String APPLY_TO_DATABASE_COMMAND_ONLINE = "gs_guc reload -c %s -D %s"; + public static final String APPLY_TO_DATABASE_COMMAND_ONLINE = "gs_guc reload -Z datanode -N all -I all -c %s"; /** * DATABASE_COMMAND_RESTART */ - public static final String DATABASE_COMMAND_RESTART = "gs_ctl restart -D %s -M primary"; + public static final String DATABASE_COMMAND_RESTART = "gs_om -t restart"; /** * FALSE diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/controller/TuningController.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/controller/TuningController.java index a8396be2963d5e49422fd8744f920d3e77af692e..e5582cbdb537a9b8508e42de54597956733394fb 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/controller/TuningController.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/controller/TuningController.java @@ -125,6 +125,17 @@ public class TuningController { return tuningService.getDatabase(clusterName); } + /** + * getDatabase + * + * @param clusterName clusterName + * @return RespBean + */ + @GetMapping("/obtain/schema") + public RespBean getSchema(@RequestParam String clusterName) { + return tuningService.getSchema(clusterName); + } + /** * getTaskclusterName * diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/domain/TrainingConfig.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/domain/TrainingConfig.java index b606c38d4b8e92eb763ef23417468672a45c040c..a11413586393a2bc012af364adb62e0c9c627d6b 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/domain/TrainingConfig.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/domain/TrainingConfig.java @@ -51,7 +51,13 @@ public class TrainingConfig { private String db; // 数据库名称 private String user; // 数据库用户 private String password; // 数据库密码 - private String ommPassword; // omm 用户密码 + private String osUser; // 安装数据库的用户 + private String chroot = "False"; // 数据库是否安装在沙箱环境 + private String distributed = "False"; // 数据库是否是分布式 + private String distDnPort; // 分布式环境下DN节点的端口号 + private String initParam; + private String workloadRestore; // 还原负载的SQL，将负载中insert对应的数据转为delete，在每轮压测完执行还原 + private String ommPassword; // 安装数据库的用户密码 private String opengaussNodePath; // opengauss_node_path private String rootPassword; // 服务器root 用户密码 private String hasPressureTab; // 是否有压测表 diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/enums/impl/ParamEnum.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/enums/impl/ParamEnum.java index 942f06d5f01631635d50b3836e97e82f5c1a4a86..326737b834c9b5b2f7aedcf67723073691472d4f 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/enums/impl/ParamEnum.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/enums/impl/ParamEnum.java @@ -105,5 +105,185 @@ public enum ParamEnum implements ParamStrategy { public String getReal() { return "否"; } + }, + AUTOVACUUM_ANALYZE_THRESHOLD { + @Override + public String getReal() { + return "否"; + } + }, + AUTOVACUUM_NAPTIME { + @Override + public String getReal() { + return "否"; + } + }, + AUTOVACUUM_VACUUM_COST_DELAY { + @Override + public String getReal() { + return "否"; + } + }, + AUTOVACUUM_VACUUM_COST_LIMIT { + @Override + public String getReal() { + return "否"; + } + }, + AUTOVACUUM_VACUUM_THRESHOLD { + @Override + public String getReal() { + return "否"; + } + }, + BGWRITER_DELAY { + @Override + public String getReal() { + return "否"; + } + }, + BGWRITER_FLUSH_AFTER { + @Override + public String getReal() { + return "否"; + } + }, + BGWRITER_LRU_MAXPAGES { + @Override + public String getReal() { + return "否"; + } + }, + CHECKPOINT_FLUSH_AFTER { + @Override + public String getReal() { + return "否"; + } + }, + CHECKPOINT_TIMEOUT { + @Override + public String getReal() { + return "否"; + } + }, + COMMIT_DELAY { + @Override + public String getReal() { + return "否"; + } + }, + COMMIT_SIBLINGS { + @Override + public String getReal() { + return "否"; + } + }, + DEADLOCK_TIMEOUT { + @Override + public String getReal() { + return "否"; + } + }, + DEFAULT_STATISTICS_TARGET { + @Override + public String getReal() { + return "否"; + } + }, + FROM_COLLAPSE_LIMIT { + @Override + public String getReal() { + return "否"; + } + }, + GEQO_EFFORT { + @Override + public String getReal() { + return "否"; + } + }, + GEQO_GENERATIONS { + @Override + public String getReal() { + return "否"; + } + }, + GEQO_POOL_SIZE { + @Override + public String getReal() { + return "否"; + } + }, + GEQO_THRESHOLD { + @Override + public String getReal() { + return "否"; + } + }, + JOIN_COLLAPSE_LIMIT { + @Override + public String getReal() { + return "否"; + } + }, + MAX_STACK_DEPTH { + @Override + public String getReal() { + return "否"; + } + }, + TEMP_FILE_LIMIT { + @Override + public String getReal() { + return "否"; + } + }, + VACUUM_COST_DELAY { + @Override + public String getReal() { + return "否"; + } + }, + WAL_WRITER_DELAY { + @Override + public String getReal() { + return "否"; + } + }, + AUTOVACUUM_ANALYZE_SCALE_FACTOR { + @Override + public String getReal() { + return "否"; + } + }, + AUTOVACUUM_VACUUM_SCALE_FACTOR { + @Override + public String getReal() { + return "否"; + } + }, + BGWRITER_LRU_MULTIPLIER { + @Override + public String getReal() { + return "否"; + } + }, + CHECKPOINT_COMPLETION_TARGET { + @Override + public String getReal() { + return "否"; + } + }, + CURSOR_TUPLE_FRACTION { + @Override + public String getReal() { + return "否"; + } + }, + GEQO_SEED { + @Override + public String getReal() { + return "否"; + } } } diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/enums/impl/TuningEnum.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/enums/impl/TuningEnum.java index 25e259e06b1359402941bc873eb92cdb7f47d8c3..bbc39a12cc42f50749160e5c2abd191deafd2f2f 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/enums/impl/TuningEnum.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/enums/impl/TuningEnum.java @@ -44,8 +44,6 @@ public enum TuningEnum implements TuningStrategy { config.getLogPath()); boolean isSysbench = CommandLineRunner.runCommand(TuningHelper.getSysBenchCommand(node, config, sysbenchCommand), FixedTuning.SYSBENCH_EXECUTE_PATH, config.getLogPath(), FixedTuning.TIME_OUT); - // 执行完Sysbench Update progress - TuningHelper.updateProcess(config.getTrainingId(), FixedTuning.PYTHON_STRATEGY_PROC); if (isSysbench) { CommandLineRunner.appendToFile("Successfully created the database for stress " + "testing and started executing python3 main.py", config.getLogPath()); @@ -65,25 +63,24 @@ public enum TuningEnum implements TuningStrategy { if (!config.getIsCustomPayloads().equals("1")) { // Use custom payload for all No TuningHelper.appendRes(config, workPath); + String jsonHelpPath = workPath + "SuperWG/DWG/jsonHelper/"; CommandLineRunner.appendToFile("Start executing jsonHelper.py", config.getLogPath()); - if (!CommandLineRunner.runCommand(FixedTuning.JSON_HELPER_COMMAND, workPath, config.getLogPath(), + if (!CommandLineRunner.runCommand(FixedTuning.JSON_HELPER_COMMAND, jsonHelpPath, config.getLogPath(), FixedTuning.TIME_OUT)) { CommandLineRunner.appendToFile("Executing jsonHelper. py failed, " + "modifying task status to failed", config.getLogPath()); TuningHelper.updateStatusFailed(config.getTrainingId()); } - TuningHelper.updateProcess(config.getTrainingId(), FixedTuning.PYTHON_ENV_PROC); + String workLoadPath = workPath + "SuperWG/DWG/src/"; CommandLineRunner.appendToFile("Start executing WorkloadGenerator.py", config.getLogPath()); - if (!CommandLineRunner.runCommand(FixedTuning.WORKLOAD_COMMAND, workPath, config.getLogPath(), + if (!CommandLineRunner.runCommand(FixedTuning.WORKLOAD_COMMAND, workLoadPath, config.getLogPath(), FixedTuning.TIME_OUT_MAIN)) { CommandLineRunner.appendToFile("Executing WorkloadGenerator. py failed, " + "modifying task status to failed", config.getLogPath()); TuningHelper.updateStatusFailed(config.getTrainingId()); } - TuningHelper.updateProcess(config.getTrainingId(), FixedTuning.PYTHON_STRATEGY_PROC); TuningHelper.execute(config, workPath, executePath); } else { - TuningHelper.updateProcess(config.getTrainingId(), FixedTuning.PYTHON_STRATEGY_PROC); TuningHelper.appendRes(config, workPath); TuningHelper.execute(config, workPath, executePath); } diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/quartz/ExecuteJob.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/quartz/ExecuteJob.java index e250e22848cdd1c187bceb9798dbc882e1ac6950..2abc54ee276af5cf6f125f85452d9e14a2d9399a 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/quartz/ExecuteJob.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/quartz/ExecuteJob.java @@ -20,8 +20,10 @@ package org.opengauss.tun.quartz; import cn.hutool.core.util.StrUtil; + import java.io.IOException; import java.io.PrintWriter; + import lombok.extern.slf4j.Slf4j; import org.opengauss.admin.common.core.domain.entity.ops.OpsJdbcDbClusterNodeEntity; import org.opengauss.admin.common.exception.ServiceException; @@ -71,8 +73,6 @@ public class ExecuteJob implements Job { createHeboEnvironment(config, workPath, recordPath); // python更新进度1 TuningHelper.updateProcess(config.getTrainingId(), FixedTuning.ENV); - // Create a sysbench environment - sysbenchEnvCreate(config, recordPath, dto.getSysbenchAddPermissions()); // write file writeConfigPy(config, executePath + FixedTuning.PYTHON_CONF); writeConfigPy(config, executePath + FixedTuning.JSONHELP_CONF); @@ -94,13 +94,7 @@ public class ExecuteJob implements Job { if (cache.includeKey(config.getClusterName())) { return; } - boolean isYum = CommandLineRunner.runCommand(FixedTuning.ENV_YUM, workPath, config.getLogPath(), - FixedTuning.TIME_OUT); - String yumSuccMsg = "Successfully executed the installation commands:" - + " - yum -y install postgresql-devel" - + " - yum install gcc libffi-devel python3-devel openssl-devel -y"; String errMsg = "Failed to create python environment, updating task status to failed"; - handleRes(isYum, yumSuccMsg, errMsg, recordPath, config); boolean isSuccess = CommandLineRunner.runCommand(FixedTuning.PYTHON3_ENV, workPath, config.getLogPath(), FixedTuning.TIME_OUT); String requirementsSuccMsg = "Successfully executed the installation commands:" @@ -125,28 +119,6 @@ public class ExecuteJob implements Job { } } - private void sysbenchEnvCreate(TrainingConfig config, String recordPath, String sysbenchAddPermissions) { - if (config.getBenchmark().equals(FixedTuning.SYSBENCH)) { - TuningHelper.record("Start detecting the sysbench environment", recordPath); - // Check the sysbench version and install if necessary - checkAndInstallSysbench(config, recordPath); - } - } - - private void checkAndInstallSysbench(TrainingConfig config, String recordPath) { - if (!isSysbenchInstalled(config, recordPath)) { - TuningHelper.record("Check the sysbench version is incorrect", recordPath); - TuningHelper.updateStatusFailed(config.getTrainingId()); - } - } - - private boolean isSysbenchInstalled(TrainingConfig config, String recordPath) { - TuningHelper.record("Check if sysbench installation was successful", recordPath); - return CommandLineRunner.runCommand(FixedTuning.CHECK_SYSBENCH_VERSION, - FixedTuning.SYSBENCH_FILE_PATH, config.getLogPath(), FixedTuning.TIME_OUT); - } - - private void writeConfigPy(TrainingConfig train, String filePath) { try (PrintWriter writer = new PrintWriter(filePath)) { TuningBuilder builder = new TuningBuilder(); @@ -158,7 +130,7 @@ public class ExecuteJob implements Job { .appendProperty("online", train.getOnline()).appendNote("# 是否进行微调, iteration为微调次数") .appendProperty("finetune", train.getFinetune()).appendNote("# 微调过程中是否允许重启") .appendProperty("restart_when_finetune", train.getRestartWhenFinetune()) - .appendNote("# 数据库待调整的配置参数文件").appendProperty("knobs_file", "'knobs_config/knobs-13.json'") + .appendNote("# 数据库待调整的配置参数文件").appendProperty("knobs_file", "'knobs_config/knobs-43.json'") .appendNote("# 进行重要性排名后, 选取的配置参数的数量") .appendProperty("ranked_knobs_number", train.getRankedKnobsNumber()) .appendProperty("method", "'HEBO'").appendProperty("log_path", "'log'") @@ -168,6 +140,7 @@ public class ExecuteJob implements Job { .appendProperty("port", train.getPort()).appendProperty("db", "'" + train.getDb() + "'") .appendProperty("user", "'" + train.getUser() + "'") .appendProperty("password", "'" + train.getPassword() + "'") + .appendProperty("os_user", "'" + train.getOsUser() + "'") .appendProperty("omm_password", "'" + train.getOmmPassword() + "'") .appendProperty("opengauss_node_path", "'" + train.getOpengaussNodePath() + "'") .appendProperty("root_password", "'" + train.getRootPassword() + "'") @@ -184,7 +157,7 @@ public class ExecuteJob implements Job { .appendSectionBreak().appendSectionBreak().appendNote("# dwg配置").appendNote("# 需要解析的schema名称") .appendProperty("schema_name", "'" + train.getSchemaName() + "'") .appendNote("# 执行gs_dump命令后在服务端保存的结果(建表语句等, 用于生成负载)") - .appendProperty("remote_cache_path", "'/home/omm/test.sql'"); + .appendProperty("remote_cache_path", "'/home/" + train.getOsUser() + "/test.sql'"); writeConfigDwg(builder, train); writer.write(builder.build()); } catch (IOException e) { @@ -203,6 +176,7 @@ public class ExecuteJob implements Job { .appendProperty("json_extract_result_path", "'" + "workloads/schema/res.json" + "'") .appendNote("# 生成负载的保存路径, 也是使用dwg压测所用的负载") .appendProperty("workload", "'" + "workloads/res.wg" + "'") + .appendProperty("workload_restore", "'workloads/restore.sql'") .appendNote("# 每个线程成功执行sql_num_print条sql后在控制台输出信息") .appendProperty("sql_num_print", train.getSqlNumPrint()) .appendSectionBreak().appendSectionBreak().appendNote("# hardware configs") @@ -215,9 +189,9 @@ public class ExecuteJob implements Job { .appendProperty("average_query_column_num", train.getAverageQueryColumnNum()) .appendProperty("group_by_ratio_if_read_sql", train.getGroupByRatioIfReadSql()) .appendProperty("order_by_desc_or_asc_if_grouped", train.getOrderByDescOrAscIfGrouped()) - .appendProperty("taskset", "False") - .appendProperty("backup", "False") - .appendProperty("restart_when_online", "False") + .appendProperty("taskset", "False").appendProperty("distributed", train.getDistributed()) + .appendProperty("chroot", train.getChroot()).appendProperty("dist_dn_port", train.getDistDnPort()) + .appendProperty("restart_when_online", "False").appendProperty("init_param", train.getInitParam()) .appendProperty("index_weight", "1") .appendProperty("primary_key_weight", "1").appendProperty("backup_path", "'/data/bak_gaussdb_data/sys/data'"); diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/quartz/ProgressJob.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/quartz/ProgressJob.java index 5cf95a9456c64a89d8b149ac9f68d3c0be2c3620..eac20db43607b1efb04fcdacb81f21e0cd8c0546 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/quartz/ProgressJob.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/quartz/ProgressJob.java @@ -44,9 +44,9 @@ import org.quartz.JobExecutionException; public class ProgressJob implements Job { private final Integer totalTaskProgress = 100; - private final Integer other = 20; + private final Integer other = 5; - private final Integer mainPy = 80; + private final Integer mainPy = 95; @Override public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/service/TuningService.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/service/TuningService.java index 338ec0ef8c208419476e55248f9df6cc28800f24..1c35008d565d451035c236e45ea60a2be1fb69d2 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/service/TuningService.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/service/TuningService.java @@ -221,4 +221,12 @@ public interface TuningService { * @return RespBean */ RespBean stopTask(List ids); + + /** + * getSchema + * + * @param clusterName clusterName + * @return RespBean + */ + RespBean getSchema(String clusterName); } diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/service/impl/TuningServiceImpl.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/service/impl/TuningServiceImpl.java index 15ff9ab655a84c0f11cd30f7d40e110bac024133..83af5c6641a6fba63248334672bc0fbf541d2eec 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/service/impl/TuningServiceImpl.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/service/impl/TuningServiceImpl.java @@ -28,6 +28,7 @@ import com.gitee.starblues.bootstrap.annotation.AutowiredType; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.jcraft.jsch.Session; + import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.File; @@ -52,6 +53,7 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; + import lombok.extern.slf4j.Slf4j; import org.opengauss.admin.common.core.domain.entity.ops.OpsHostEntity; import org.opengauss.admin.common.core.domain.entity.ops.OpsHostUserEntity; @@ -96,7 +98,6 @@ import org.opengauss.tun.utils.IdUtils; import org.opengauss.tun.utils.JschUtil; import org.opengauss.tun.utils.SchedulerUtils; import org.opengauss.tun.utils.file.FileLoader; -import org.opengauss.tun.utils.jdbc.JdbcUtil; import org.opengauss.tun.utils.response.RespBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -165,7 +166,7 @@ public class TuningServiceImpl implements TuningService { // Obtain node information OpsJdbcDbClusterNodeEntity node = getNode(config.getClusterName()); // Verify - validate(config, workPath); + validate(config, FixedTuning.SYSBENCH_FILE_PATH); // Parsing custom payloads AssertUtil.save(1, configMpper.insert(getConfig(config, node)), "Training task save failed"); TuningDto dto = new TuningDto(config, executePath, workPath, node, sysbenchCommand, sysbenchAddPermissions); @@ -218,8 +219,18 @@ public class TuningServiceImpl implements TuningService { private void checkTuning(TrainingConfig config, String workPath) { // check Python version - AssertUtil.isTrue(CommandLineRunner.runCommand(FixedTuning.CHECK_PYTHON_VERSION, + AssertUtil.isTrue(!CommandLineRunner.runCommand(FixedTuning.CHECK_PYTHON_VERSION, workPath, "", 1), "Python 3 version must be greater than or equal to 3.7.0"); + // If it is sysbench, check if sysbench is installed + if (config.getBenchmark().equals("sysbench")) { + AssertUtil.isTrue(!CommandLineRunner.runCommand(FixedTuning.CHECK_SYSBENCH, + workPath, "", 1), "The current pressure testing mode is sysbench. " + + "Please install sysbench"); + } + // Check dependencies postgresql-devel + String rely = CommandLineRunner.runCommand(FixedTuning.CHECK_RELY, + workPath, 1).trim(); + AssertUtil.isTrue(StrUtil.isNotEmpty(rely), rely); List configs = configMpper.selectList(new QueryWrapper() .eq("cluster_name", config.getClusterName())); TrainingConfig config1 = configs.stream().filter(item -> item.getOnline().equals(FixedTuning.FALSE)) @@ -237,20 +248,19 @@ public class TuningServiceImpl implements TuningService { private TrainingConfig getConfig(TrainingConfig config, OpsJdbcDbClusterNodeEntity node) { String id = IdUtils.getId(); config.setTrainingId(id); + // Set the connection information for the database config.setHost(node.getIp()); config.setPort(node.getPort()); config.setUser(node.getUsername()); config.setPassword(node.getPassword()); - Optional optional = getLinuxConfig(node.getIp()); - AssertUtil.isTrue(!optional.isPresent(), "Failed to obtain open gauss database server information"); - config.setRootPassword(getLinuxConfig(node.getIp()).get().getPassword()); - // 校验数据库,schema信息 - Connection connection = getConnection(config.getClusterName(), config.getDb()); - List schemas = JdbcUtil.getAllSchemas(connection, config.getDb()); - AssertUtil.isTrue(!schemas.contains(config.getSchemaName().trim()), - "The schema does not exist in the database"); - // 校验 linxu omm 用户连接信息 - checkConnection(config, optional.get()); + // Set server root connection information + Optional rootOpt = getLinuxConfig(node.getIp(), "root"); + AssertUtil.isTrue(!rootOpt.isPresent(), "Failed to obtain root server information"); + config.setRootPassword(rootOpt.get().getPassword()); + // Set the user connection information for installing the database + Optional dbOpt = getLinuxConfig(node.getIp(), config.getOsUser()); + AssertUtil.isTrue(!dbOpt.isPresent(), "Failed to obtain opengauss database server information"); + config.setOmmPassword(dbOpt.get().getPassword()); config.setStartTime(DateUtil.getTimeNow()); config.setLogPath(getTuningLogPath(id)); List fileNames = parsingCustomPayloads(config, config.getLogPath()); @@ -275,18 +285,6 @@ public class TuningServiceImpl implements TuningService { return tuningRecordsPath; } - private void checkConnection(TrainingConfig trainingConfig, LinuxConfig config) { - // ssh 连接 - Session ommSession = JschUtil.obtainSession(LinuxConfig.builder().host(trainingConfig.getHost()) - .port(config.getPort()).userName("omm").password(trainingConfig.getOmmPassword()).build()); - // 校验路径 - Session rootSession = JschUtil.obtainSession(config); - AssertUtil.isTrue(!JschUtil.isPathExists(rootSession, trainingConfig.getOpengaussNodePath()), - "The data path does not exist"); - JschUtil.closeSession(ommSession); - JschUtil.closeSession(rootSession); - } - private SysConfig buildSysConfig(DbTypeEnum dbType, String url, String username, String password) { return SysConfig.builder() .userName(username) @@ -308,10 +306,10 @@ public class TuningServiceImpl implements TuningService { return node.get(); } - private Optional getLinuxConfig(String host) { + private Optional getLinuxConfig(String host, String user) { List entities = hostService.list(); List userEntities = hostUserService.list(); - Map userMap = userEntities.stream().filter(item -> item.getUsername().equals("root")) + Map userMap = userEntities.stream().filter(item -> item.getUsername().equals(user)) .collect(Collectors.toMap(OpsHostUserEntity::getHostId, Function.identity())); return entities.stream() .filter(entity -> entity.getPublicIp().equals(host)) @@ -342,13 +340,23 @@ public class TuningServiceImpl implements TuningService { @Override public RespBean getDatabase(String clusterName) { try (Connection connection = getConnection(clusterName, "")) { - List databases = retrieveDatabases(connection); + List databases = retrieveDatabases(connection, FixedTuning.SEARCH_DATABASE, FixedTuning.SEARCH_RES); return RespBean.success("success", databases); } catch (SQLException e) { throw new ServiceException(e.getMessage()); } } + @Override + public RespBean getSchema(String clusterName) { + try (Connection connection = getConnection(clusterName, "")) { + List schema = retrieveDatabases(connection, FixedTuning.SEARCH, FixedTuning.RES); + return RespBean.success("success", schema); + } catch (SQLException e) { + throw new ServiceException(e.getMessage()); + } + } + private Connection getConnection(String clusterName, String dbName) { OpsJdbcDbClusterNodeEntity node = getNode(clusterName); if (StrUtil.isNotEmpty(dbName)) { @@ -359,12 +367,12 @@ public class TuningServiceImpl implements TuningService { return ConnectionUtils.getConnection(gaussConfig); } - private List retrieveDatabases(Connection connection) throws SQLException { + private List retrieveDatabases(Connection connection, String sql, String searches) throws SQLException { List databases = new ArrayList<>(); try (Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery(FixedTuning.SEARCH_DATABASE)) { + ResultSet rs = stmt.executeQuery(sql)) { while (rs.next()) { - String dbName = rs.getString(FixedTuning.SEARCH_RES); + String dbName = rs.getString(searches); databases.add(dbName); } } @@ -452,48 +460,43 @@ public class TuningServiceImpl implements TuningService { // Obtain information on the server where the OpenGaussian database is located and obtain a session TrainingConfig config = configMpper.selectById(applyVo.getTrainingId()); AssertUtil.isTrue(ObjectUtil.isEmpty(config), "Training record does not exist"); - Optional optional = getLinuxConfig(config.getHost()); + Optional optional = getLinuxConfig(config.getHost(), config.getOsUser()); AssertUtil.isTrue(!optional.isPresent(), "Failed to obtain openGauss database server information"); // 获取omm用户session - LinuxConfig ommUser = optional.get(); - ommUser.setUserName("omm"); - ommUser.setPassword(config.getOmmPassword()); - Session session = JschUtil.obtainSession(ommUser); - String command = getOptCommand(applyVo.getIsOptimization(), applyVo.getData(), config) + FixedTuning.DETERMINE; + LinuxConfig dbUser = optional.get(); + dbUser.setUserName(config.getOsUser()); + dbUser.setPassword(config.getOmmPassword()); + Session session = JschUtil.obtainSession(dbUser); + boolean isAll = applyVo.getData().stream().allMatch(s -> s.getRestart().equalsIgnoreCase("否")); + boolean isOnLine = FixedTuning.TRUE.equals(config.getOnline()); + String command = getOptCommand(applyVo.getIsOptimization(), + applyVo.getData(), isOnLine) + FixedTuning.DETERMINE; String res = JschUtil.executeCommand(session, command); String message = String.format("Execute command:%s The result is:%s", command, res); - if (res.contains(FixedTuning.SUCCESS_INSTALL)) { - // The set command requires restarting the database - if (command.contains("set")) { - String reStart = String.format(FixedTuning.DATABASE_COMMAND_RESTART, config.getOpengaussNodePath()); - String restartResult = JschUtil.executeCommand(session, reStart); - log.info("The result of restarting the database is-->{}", restartResult); - } - return RespBean.success("success", message); - } else { - return RespBean.error(message); + if (res.contains(FixedTuning.SUCCESS_INSTALL) && !isAll && !isOnLine) { + String restartResult = JschUtil.executeCommand(session, FixedTuning.DATABASE_COMMAND_RESTART); + log.info("The result of restarting the database is-->{}", restartResult); } + return RespBean.success("success", message); } - private String getOptCommand(Boolean isOptimization, List show, TrainingConfig config) { + private String getOptCommand(Boolean isOpt, List show, boolean isOnLine) { StringBuilder commandBuilder = new StringBuilder(); - int index = 0; - int size = show.size(); for (ParameterShow param : show) { String parameterName = param.getParameterName(); - String parameterValues = isOptimization + String parameterValues = isOpt ? param.getSuggestedParameterValues() : param.getInitialParameterValues(); - commandBuilder.append("\"").append(parameterName).append("=").append(parameterValues).append("\""); - if (index < size - 1) { + if (StrUtil.isNotEmpty(parameterValues)) { + commandBuilder.append("\"").append(parameterName).append("=").append(parameterValues).append("\""); commandBuilder.append(" -c "); } - index++; } - boolean isAll = show.stream().allMatch(s -> s.getRestart().equalsIgnoreCase("否")); String core = commandBuilder.toString(); - String applyToDatabaseCommand = isAll + String applyToDatabaseCommand = isOnLine ? FixedTuning.APPLY_TO_DATABASE_COMMAND_ONLINE : FixedTuning.APPLY_TO_DATABASE_COMMAND_OFFLINE; - return String.format(applyToDatabaseCommand, core, config.getOpengaussNodePath()); + String command = String.format(applyToDatabaseCommand, core); + int lastCPos = command.lastIndexOf(" -c"); + return command.substring(0, lastCPos).trim(); } private void setLogDetails(TuningLog log) { diff --git a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/utils/CommandLineRunner.java b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/utils/CommandLineRunner.java index c3ac73f41aab4e6091345b7c3a7c1755d662e805..c5bb26200f12478a01c67ec222c7d9322d93852e 100644 --- a/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/utils/CommandLineRunner.java +++ b/plugins/intelligent-parameter-tuning/src/main/java/org/opengauss/tun/utils/CommandLineRunner.java @@ -20,6 +20,7 @@ package org.opengauss.tun.utils; import cn.hutool.core.util.StrUtil; + import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; @@ -30,6 +31,7 @@ import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.StringJoiner; import java.util.concurrent.TimeUnit; + import lombok.extern.slf4j.Slf4j; import org.opengauss.tun.common.FixedTuning; @@ -70,13 +72,45 @@ public class CommandLineRunner { result.append(output).append(StrUtil.LF).append(error); appendToFile(result.toString(), writePath); String res = result.toString(); - return res.contains(FixedTuning.SUCCESS_INSTALL) || res.equals(FixedTuning.BOOL_TRUE); + if (res.contains(FixedTuning.SUCCESS_INSTALL)) { + return true; + } + if (res.trim().equals(FixedTuning.BOOL_TRUE)) { + return true; + } + return false; } catch (IOException | InterruptedException exception) { + log.error("adsagdfshgfdshdsfhfdh"); log.error("Command execution failed: {}", exception.getMessage()); return false; } } + /** + * runCommand + * + * @param command command + * @param filePath filePath + * @param timeOut timeOut + * @return boolean + */ + public static String runCommand(String command, String filePath, long timeOut) { + try { + ProcessBuilder builder = new ProcessBuilder("bash", "-c", command); + builder.directory(new File(filePath)); + Process process = builder.start(); + String output = readFromStream(process.getInputStream()); + String error = readFromStream(process.getErrorStream()); + process.waitFor(timeOut, TimeUnit.MINUTES); + StringBuilder result = new StringBuilder(); + result.append(output).append(StrUtil.LF).append(error); + return result.toString(); + } catch (IOException | InterruptedException exception) { + log.error("Command execution failed: {}", exception.getMessage()); + return ""; + } + } + private static String readFromStream(InputStream stream) throws IOException { StringJoiner joiner = new StringJoiner(StrUtil.LF); try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { diff --git a/plugins/intelligent-parameter-tuning/src/main/resources/db/main.sql b/plugins/intelligent-parameter-tuning/src/main/resources/db/main.sql index 70eaf60439ee37da63ec4d73691dc612eec9e1b8..451ff375a7681ceb5c4bb94a0d9f2a01175e0201 100644 --- a/plugins/intelligent-parameter-tuning/src/main/resources/db/main.sql +++ b/plugins/intelligent-parameter-tuning/src/main/resources/db/main.sql @@ -7,6 +7,12 @@ port TEXT, db TEXT, user TEXT, password TEXT, +os_user TEXT, +chroot TEXT, +init_param TEXT, +workload_restore TEXT, +dist_dn_port TEXT, +distributed TEXT, omm_password TEXT, opengauss_node_path TEXT, root_password TEXT, diff --git a/plugins/intelligent-parameter-tuning/src/main/resources/tun/opengauss_tuning.zip b/plugins/intelligent-parameter-tuning/src/main/resources/tun/opengauss_tuning.zip index b4021696ae4cdd9934734f7e50285688c7bcf032..63f98751a339261550229f5b8e947015a3d2cee6 100644 Binary files a/plugins/intelligent-parameter-tuning/src/main/resources/tun/opengauss_tuning.zip and b/plugins/intelligent-parameter-tuning/src/main/resources/tun/opengauss_tuning.zip differ diff --git a/plugins/intelligent-parameter-tuning/web-ui/src/api/ops/index.ts b/plugins/intelligent-parameter-tuning/web-ui/src/api/ops/index.ts index f6981fa2b07c4a7c5cee029ccb24146fd28d129f..6e5f1bd9fd5f354b8c5420e5a073e76aa26195d6 100644 --- a/plugins/intelligent-parameter-tuning/web-ui/src/api/ops/index.ts +++ b/plugins/intelligent-parameter-tuning/web-ui/src/api/ops/index.ts @@ -333,6 +333,12 @@ export const getAlldbList = (data: string) => { }) } +export const getSchemaNameList = (data: string) => { + return axios.get('intelligent/tuning/obtain/schema', { + params: data + }) +} + export const getFiles = (data: KeyValue) => { return axios.get(`intelligent/tuning/obtain/file/${data.trainingId}`) } diff --git a/plugins/intelligent-parameter-tuning/web-ui/src/locale/locale/en.json b/plugins/intelligent-parameter-tuning/web-ui/src/locale/locale/en.json index 2bf272bffb7c76f63ed3268d9b223d211f52d483..028fe8364f41c69d76d8322224fbb97792e10b18 100644 --- a/plugins/intelligent-parameter-tuning/web-ui/src/locale/locale/en.json +++ b/plugins/intelligent-parameter-tuning/web-ui/src/locale/locale/en.json @@ -33,8 +33,7 @@ "5mpn60ejri12": "Cluster name", "5mpn60ejri13": "Database", "5mpn60ejri14": "schema", - "5mpn60ejri15": "Data path", - "5mpn60ejri16": "Omm user password", + "5mpn60ejri15": "Database user", "5mpn60ejri17": "Is there a pressure gauge available", "5mpn60fjra17": "yes", "5mpn60fjrb17": "no", @@ -63,9 +62,8 @@ "6mpn60ejri26": "The number of pressure measurement parameters cannot be empty", "7mpn60ejri12": "Please select a cluster name", "7mpn60ejri13": "Please select a database", - "7mpn60ejri14": "Please enter a schema", - "7mpn60ejri15": "Please enter the data path", - "7mpn60ejri16": "Please enter the OMM user password", + "7mpn60ejri14": "Please select a schema", + "7mpn60ejri15": "Please enter the Database user", "7mpn60ejri18": "Please enter the number of tables", "7mpn60ejri19": "Please enter the size of the table data", "7mpn60ejri21": "Please select pressure testing mode", diff --git a/plugins/intelligent-parameter-tuning/web-ui/src/locale/locale/zh-cn.json b/plugins/intelligent-parameter-tuning/web-ui/src/locale/locale/zh-cn.json index 9f58ce8be59dd03167e678768ac9672f368f63ae..8512a58f6fd0a9432a32d89e3decbca11d7f6406 100644 --- a/plugins/intelligent-parameter-tuning/web-ui/src/locale/locale/zh-cn.json +++ b/plugins/intelligent-parameter-tuning/web-ui/src/locale/locale/zh-cn.json @@ -33,8 +33,7 @@ "5mpn60ejri12": "集群名称", "5mpn60ejri13": "数据库", "5mpn60ejri14": "schema", - "5mpn60ejri15": "数据路径", - "5mpn60ejri16": "omm用户密码", + "5mpn60ejri15": "数据库安装用户", "5mpn60ejri17": "是否已有压测表", "5mpn60fjra17": "是", "5mpn60fjrb17": "否", @@ -51,8 +50,8 @@ "6mpn60ejri12": "集群名称不能为空", "6mpn60ejri13": "数据库不能为空", "6mpn60ejri14": "schema不能为空", - "6mpn60ejri15": "数据路径不能为空", - "6mpn60ejri16": "omm用户密码不能为空", + "6mpn60ejri15": "数据库安装用户不能为空", + "6mpn60ejri16": "os用户密码不能为空", "6mpn60ejri18": "表数量不能为空", "6mpn60ejri19": "表数据量大小不能为空", "6mpn60ejri21": "压测模式不能为空", @@ -63,9 +62,8 @@ "6mpn60ejri26": "压测参数数量不能为空", "7mpn60ejri12": "请选择集群名称", "7mpn60ejri13": "请选择数据库", - "7mpn60ejri14": "请输入schema", - "7mpn60ejri15": "请输入数据路径", - "7mpn60ejri16": "请输入omm用户密码", + "7mpn60ejri14": "请选择schema", + "7mpn60ejri15": "请输入数据库安装用户", "7mpn60ejri18": "请输入表数量", "7mpn60ejri19": "请输入表数据量大小", "7mpn60ejri21": "请选择压测模式", diff --git a/plugins/intelligent-parameter-tuning/web-ui/src/views/ops/install/components/InstallStep.vue b/plugins/intelligent-parameter-tuning/web-ui/src/views/ops/install/components/InstallStep.vue index 8b0f1d5b0bdf7177c23f6d7939961483748c8456..87bc7fbaf5fc6c55e24f2540d280d1be4282862c 100644 --- a/plugins/intelligent-parameter-tuning/web-ui/src/views/ops/install/components/InstallStep.vue +++ b/plugins/intelligent-parameter-tuning/web-ui/src/views/ops/install/components/InstallStep.vue @@ -4,19 +4,23 @@ v-if="installStore.benchMark === BenchMarkEnum.SYSBENCH " ref="offlineInstallRef" @selectdbList="getAlldbName" + @selectschemaNameList="getAllschemaName" :dbList="dbList" + :schemaNameList="schemaNameList" />