diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckContext.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckContext.java index 9f0bc4ffba682abcd1219eef8cf64ddedd26c0ec..007ed29da09a733135020189ad5d8d2c5f49d26c 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckContext.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckContext.java @@ -127,8 +127,13 @@ public class SliceCheckContext { sliceCheckResultManager.addTableStructureDiffResult(slice, result); } + /** + * save process history logging + * + * @param slice slice + */ public void saveProcessHistoryLogging(SliceVo slice) { - processLogService.saveProcessHistoryLogging(slice.getTable(), slice.getNo()); + processLogService.saveProcessHistoryLogging(slice.getTable(), slice.getTotal(), slice.getNo()); } /** diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/service/ProcessLogService.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/service/ProcessLogService.java index 078a52a8b06c5ee697a08756e790cbd64f48512c..605f867d62b9072052bf1fea1da9f6663ab223ae 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/service/ProcessLogService.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/service/ProcessLogService.java @@ -65,9 +65,17 @@ public class ProcessLogService { } - public void saveProcessHistoryLogging(String tableName, int order) { + /** + * save process history logging + * + * @param tableName table name + * @param size slice size + * @param order slice order + */ + public void saveProcessHistoryLogging(String tableName, int size, int order) { ProcessingLog processingLog = new ProcessingLog().setEndpoint(ConfigCache.getEndPoint()) .setTable(tableName) + .setSize(size) .setOrder(order) .setFinishedTime(LocalDateTime.now()); if (logRootPath == null) { @@ -110,11 +118,12 @@ public class ProcessLogService { @Data @Accessors(chain = true) - @JSONType(orders = {"endpoint", "table", "order", "finishedTime"}) + @JSONType(orders = {"endpoint", "table", "size", "order", "finishedTime"}) class ProcessingLog { Endpoint endpoint; String table; int order; + int size; LocalDateTime finishedTime; } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java index 47b096703615c48bdeb5e71d71a57faab6c73676..cc6ce38b480f8ba68b0527bce889a771f866b2f4 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java @@ -27,11 +27,13 @@ import org.opengauss.datachecker.common.entry.enums.ErrorCode; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.opengauss.datachecker.common.exception.ExtractException; import org.opengauss.datachecker.common.util.LogUtils; import org.opengauss.datachecker.common.util.LongHashFunctionWrapper; import org.opengauss.datachecker.extract.cache.MetaDataCache; import org.opengauss.datachecker.extract.data.access.DataAccessService; import org.opengauss.datachecker.extract.service.RuleAdapterService; +import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.stereotype.Service; import javax.annotation.Resource; @@ -105,15 +107,20 @@ public class BaseDataService { * @return table name list */ public List bdsQueryTableMetadataList() { - List metadataList = dataAccessService.dasQueryTableMetadataList(); - return metadataList.stream().filter(meta -> { - boolean isChecking = ruleAdapterService.filterTableByRule(meta.getTableName()); - if (isChecking) { - tableNameList.add(meta.getTableName()); - } - meta.setExistTableRows(dataAccessService.tableExistsRows(meta.getTableName())); - return isChecking; - }).collect(Collectors.toList()); + try { + List metadataList = dataAccessService.dasQueryTableMetadataList(); + return metadataList.stream().filter(meta -> { + boolean isChecking = ruleAdapterService.filterTableByRule(meta.getTableName()); + if (isChecking) { + tableNameList.add(meta.getTableName()); + } + meta.setExistTableRows(dataAccessService.tableExistsRows(meta.getTableName())); + return isChecking; + }).collect(Collectors.toList()); + } catch (BadSqlGrammarException ex) { + log.error("{}query table metadata list error {}", ErrorCode.EXECUTE_QUERY_SQL, ex.getMessage()); + throw new ExtractException("query table metadata list error " + ex.getMessage()); + } } /** diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java index 2e638e5f2afabe100f6a2a8370b0586b4bb45aaa..ef246a3f89e5721358f5989fd246b2be12f2fd4a 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java @@ -126,7 +126,11 @@ public class MysqlDataAccessService extends AbstractDataAccessService { @Override public boolean tableExistsRows(String tableName) { - return mysqlMetaDataMapper.tableExistsRows(properties.getSchema(), tableName); + return mysqlMetaDataMapper.tableExistsRows(properties.getSchema(), escape(tableName)); + } + + private String escape(String tableName) { + return "`" + tableName + "`"; } @Override diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java index 4840d83b730a85f9228068ea7954d4930c8a14a2..ff83211a2651e8752d46d9d637bf14d1a83606b9 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java @@ -172,7 +172,11 @@ public class OpgsDataAccessService extends AbstractDataAccessService { @Override public boolean tableExistsRows(String tableName) { - return opgsMetaDataMapper.tableExistsRows(properties.getSchema(), tableName); + return opgsMetaDataMapper.tableExistsRows(properties.getSchema(), compatibilityEscape(tableName)); + } + + private String compatibilityEscape(String tableName) { + return isOgCompatibilityB ? "`" + tableName + "`" : "\"" + tableName + "\""; } @Override diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/load/ExtractEnvironmentContext.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/load/ExtractEnvironmentContext.java index 9aacf19853ff4a6c36372b963efb76de20ada45d..570ef5b80564559a5b3cb3cfad9710c0688094e4 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/load/ExtractEnvironmentContext.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/load/ExtractEnvironmentContext.java @@ -18,7 +18,9 @@ package org.opengauss.datachecker.extract.load; import org.apache.logging.log4j.Logger; import org.opengauss.datachecker.common.config.ConfigCache; import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.enums.ErrorCode; import org.opengauss.datachecker.common.entry.extract.MetadataLoadProcess; +import org.opengauss.datachecker.common.exception.ExtractException; import org.opengauss.datachecker.common.service.ShutdownService; import org.opengauss.datachecker.common.util.LogUtils; import org.opengauss.datachecker.common.util.ThreadUtil; @@ -57,20 +59,25 @@ public class ExtractEnvironmentContext { */ @Async public void loadDatabaseMetaData() { - LogUtils.info(log,"extract database loader start"); - metaDataService.loadMetaDataOfSchemaCache(); - ThreadUtil.sleepHalfSecond(); - MetadataLoadProcess metadataLoadProcess = metaDataService.getMetadataLoadProcess(); - while (!metadataLoadProcess.isLoadSuccess()) { - LogUtils.info(log,"extract service load table meta ={}", metadataLoadProcess.getLoadCount()); - ThreadUtil.sleepOneSecond(); - metadataLoadProcess = metaDataService.getMetadataLoadProcess(); + LogUtils.info(log, "extract database loader start"); + try { + metaDataService.loadMetaDataOfSchemaCache(); + ThreadUtil.sleepHalfSecond(); + MetadataLoadProcess metadataLoadProcess = metaDataService.getMetadataLoadProcess(); + while (!metadataLoadProcess.isLoadSuccess()) { + LogUtils.info(log, "extract service load table meta ={}", metadataLoadProcess.getLoadCount()); + ThreadUtil.sleepOneSecond(); + metadataLoadProcess = metaDataService.getMetadataLoadProcess(); + } + Endpoint endPoint = ConfigCache.getEndPoint(); + checkingFeignClient.refreshLoadMetadataCompleted(endPoint); + if (metaDataService.mdsIsCheckTableEmpty(false)) { + shutdownService.shutdown("load table metadata cache is empty!"); + } + LogUtils.info(log, "extract service load table meta ={} , success", metadataLoadProcess.getLoadCount()); + } catch (ExtractException ex) { + LogUtils.error(log, "{}extract database loader error", ErrorCode.CHECK_START_ERROR, ex); + shutdownService.shutdown("extract database loader error!"); } - Endpoint endPoint = ConfigCache.getEndPoint(); - checkingFeignClient.refreshLoadMetadataCompleted(endPoint); - if (metaDataService.mdsIsCheckTableEmpty(false)) { - shutdownService.shutdown("load table metadata cache is empty!"); - } - LogUtils.info(log,"extract service load table meta ={} , success", metadataLoadProcess.getLoadCount()); } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/SliceProcessorContext.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/SliceProcessorContext.java index c061f28938d2b122e4d897f561291ea77a928624..4e2438d4085d1fbcf04f436f86dfe5dcf7b1bc35 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/SliceProcessorContext.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/SliceProcessorContext.java @@ -68,8 +68,13 @@ public class SliceProcessorContext { private ThreadPoolTaskExecutor sliceSendExecutor; private SliceStatusFeedbackService sliceStatusFeedbackService; + /** + * save processing + * + * @param slice slice + */ public void saveProcessing(SliceVo slice) { - processLogService.saveProcessHistoryLogging(slice.getTable(), slice.getNo()); + processLogService.saveProcessHistoryLogging(slice.getTable(), slice.getTotal(), slice.getNo()); } /**