diff --git a/config/application-sink.yml b/config/application-sink.yml index 3003894e126a9aa484cca5fda9948ac3effa04cb..6fbad48502cd532177585bf407145f1e7a98de84 100644 --- a/config/application-sink.yml +++ b/config/application-sink.yml @@ -44,6 +44,6 @@ spring: validation-query-timeout: 10000 connection-error-retry-attempts: 0 break-after-acquire-failure: true - max-wait: 6000 + max-wait: 600000 keep-alive: true min-evictable-idle-time-millis: 600000 \ No newline at end of file diff --git a/config/application-source.yml b/config/application-source.yml index 2c3d136f2564d3b29c8c04645cc5abf43577ffc6..e23314958daeb9405a2e3c86cf346028e49f73d6 100644 --- a/config/application-source.yml +++ b/config/application-source.yml @@ -51,6 +51,6 @@ spring: validation-query-timeout: 10000 connection-error-retry-attempts: 0 break-after-acquire-failure: true - max-wait: 6000 + max-wait: 600000 keep-alive: true min-evictable-idle-time-millis: 600000 \ No newline at end of file diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/load/CheckDatabaseLoader.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/load/CheckDatabaseLoader.java index 06b632bc1871cc808094fea5a8b7903024426db0..60ee6c3ed31e4377865ac879da86c4404dfe95e4 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/load/CheckDatabaseLoader.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/load/CheckDatabaseLoader.java @@ -73,9 +73,15 @@ public class CheckDatabaseLoader extends AbstractCheckLoader { ConfigCache.put(ConfigConstants.DATA_CHECK_SOURCE_DATABASE, sourceConfig.getDatabase()); ConfigCache.put(ConfigConstants.DATA_CHECK_SINK_DATABASE, sinkConfig.getDatabase()); ConfigCache.put(ConfigConstants.LOWER_CASE_TABLE_NAMES, sinkConfig.getDatabase().getLowercaseTableNames()); + ConfigCache.put(ConfigConstants.MAXIMUM_TABLE_SLICE_SIZE, getMaxTableSliceConfig(sourceConfig, sinkConfig)); LogUtils.info(log, "check service load database configuration success."); } + private static int getMaxTableSliceConfig(ExtractConfig sourceConfig, ExtractConfig sinkConfig) { + int max = Math.max(sourceConfig.getMaxSliceSize(), sinkConfig.getMaxSliceSize()); + return max == 0 ? ConfigConstants.MAXIMUM_TABLE_SLICE_DEFAULT_VALUE : max; + } + private void checkDatabaseLowerCaseTableNames(Database source, Database sink) { Assert.notNull(source, "source database config can't be null"); Assert.notNull(sink, "sink database config can't be null"); diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/AbstractCheckDiffResultBuilder.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/AbstractCheckDiffResultBuilder.java index 18cced281bb0bab56735c4c3bd7ea3a121b91727..a0f8b027cbd22b8bd24299e05295e7dc73793c36 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/AbstractCheckDiffResultBuilder.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/AbstractCheckDiffResultBuilder.java @@ -44,11 +44,15 @@ import java.util.stream.Collectors; public abstract class AbstractCheckDiffResultBuilder> { private static final Logger log = LogUtils.getLogger(); - private static final int MAX_DIFF_REPAIR_SIZE = 5000; + private static final int MAX_DIFF_REPAIR_SIZE = 100; private String table; private int partitions; private int rowCount; + private int totalRepair; + private int insertTotal; + private int updateTotal; + private int deleteTotal; private int errorRate; private int sno; private long beginOffset; @@ -102,6 +106,42 @@ public abstract class AbstractCheckDiffResultBuilder insert, List update, List delete) { if (Objects.nonNull(insert)) { - this.keyInsert.addAll(insert); - this.keyInsertSet.addAll(insert.stream().map(Difference::getKey).collect(Collectors.toSet())); + this.keyInsert.addAll(insert.stream().limit(MAX_DIFF_REPAIR_SIZE).collect(Collectors.toList())); + this.keyInsertSet.addAll( + insert.stream().map(Difference::getKey).limit(MAX_DIFF_REPAIR_SIZE).collect(Collectors.toSet())); } if (Objects.nonNull(update)) { - this.keyUpdate.addAll(update); - this.keyUpdateSet.addAll(update.stream().map(Difference::getKey).collect(Collectors.toSet())); + this.keyUpdate.addAll(update.stream().limit(MAX_DIFF_REPAIR_SIZE).collect(Collectors.toList())); + this.keyUpdateSet.addAll( + update.stream().map(Difference::getKey).limit(MAX_DIFF_REPAIR_SIZE).collect(Collectors.toSet())); } if (Objects.nonNull(delete)) { - this.keyDelete.addAll(delete); - this.keyDeleteSet.addAll(delete.stream().map(Difference::getKey).collect(Collectors.toSet())); + this.keyDelete.addAll(delete.stream().limit(MAX_DIFF_REPAIR_SIZE).collect(Collectors.toList())); + this.keyDeleteSet.addAll( + delete.stream().map(Difference::getKey).limit(MAX_DIFF_REPAIR_SIZE).collect(Collectors.toSet())); } diffSort.sort(this.keyInsert); diffSort.sort(this.keyUpdate); @@ -331,14 +374,11 @@ public abstract class AbstractCheckDiffResultBuilder 0 ? (totalRepair * 100 / rowCount) : 0; - if (totalRepair <= MAX_DIFF_REPAIR_SIZE || curErrorRate <= errorRate) { + if (totalRepair <= MAX_DIFF_REPAIR_SIZE) { return true; } else { - log.info("check table[{}.{}] diff-count={},error-rate={}%, error is too large ,not to build repair dml", - schema, table, totalRepair, curErrorRate); + log.info("check table[{}.{}] diff-count={}, error is too large ,not to build repair dml", schema, table, + totalRepair); return false; } } diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckDiffResult.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckDiffResult.java index 2bf8f142ed7c7c2c5d5e9921e6ea395111455598..27a923efb717464306ea488700565f38a804b9be 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckDiffResult.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckDiffResult.java @@ -48,10 +48,10 @@ import static org.opengauss.datachecker.check.modules.check.CheckResultConstants * @since ：11 */ @Data -@JSONType( - orders = {"process", "schema", "table", "topic", "checkMode", "result", "message", "error", "startTime", "endTime", - "keyInsertSet", "keyUpdateSet", "keyDeleteSet", "keyInsert", "keyUpdate", "keyDelete"}, - ignores = {"sno", "partitions", "beginOffset", "totalRepair", "buildRepairDml", "isBuildRepairDml", "rowCondition"}) +@JSONType(orders = { + "process", "schema", "table", "topic", "checkMode", "result", "message", "error", "startTime", "endTime", + "keyInsertSet", "keyUpdateSet", "keyDeleteSet", "keyInsert", "keyUpdate", "keyDelete" +}, ignores = {"sno", "partitions", "beginOffset", "totalRepair", "buildRepairDml", "isBuildRepairDml", "rowCondition"}) public class CheckDiffResult { private String process; private String schema; @@ -62,7 +62,10 @@ public class CheckDiffResult { private int partitions; private long beginOffset; private long rowCount; - private int totalRepair; + private long totalRepair; + private long insertTotal; + private long updateTotal; + private long deleteTotal; private boolean isTableStructureEquals; private CheckMode checkMode; private LocalDateTime startTime; @@ -115,8 +118,10 @@ public class CheckDiffResult { keyInsert = builder.getKeyInsert(); keyUpdate = builder.getKeyUpdate(); keyDelete = builder.getKeyDelete(); - totalRepair = keyUpdateSet.size() + keyInsertSet.size() + keyDeleteSet.size(); - totalRepair = totalRepair + keyUpdate.size() + keyInsert.size() + this.keyDelete.size(); + insertTotal = builder.getInsertTotal(); + updateTotal = builder.getUpdateTotal(); + deleteTotal = builder.getDeleteTotal(); + totalRepair = builder.getTotalRepair(); resultAnalysis(builder.isNotLargeDiffKeys()); } else { initEmptyCollections(); @@ -142,8 +147,8 @@ public class CheckDiffResult { private void resultAnalysis(boolean isNotLargeDiffKeys) { if (Objects.nonNull(rowCondition)) { - message = - String.format(CHECKED_ROW_CONDITION, schema, table, rowCondition.getStart(), rowCondition.getOffset()); + message = String.format(CHECKED_ROW_CONDITION, schema, table, rowCondition.getStart(), + rowCondition.getOffset()); } else { message = String.format(CHECKED_PARTITIONS, schema, table, sno); } @@ -158,8 +163,7 @@ public class CheckDiffResult { } } else { result = RESULT_FAILED; - message += String.format(FAILED_MESSAGE, keyInsertSet.size() + keyInsert.size(), - keyUpdateSet.size() + keyUpdate.size(), keyDeleteSet.size() + keyDelete.size()); + message += String.format(FAILED_MESSAGE, insertTotal, updateTotal, deleteTotal); if (totalRepair > 0 && !isNotLargeDiffKeys) { message += CHECKED_DIFF_TOO_LARGE; } diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckResultConstants.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckResultConstants.java index b8f616d566529fd858e2739eb8311425742091cd..5c43cc5fa682afae5cfa2044249c5739aaa2b2ac 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckResultConstants.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckResultConstants.java @@ -27,6 +27,12 @@ public interface CheckResultConstants { String FAILED_LOG_NAME = "failed.log"; String CSV_FAILED_DETAIL_NAME = "failed_detail.log"; String RESULT_SUCCESS = "success"; + + /** + * The display quantity of the verification result failure records + */ + int MAX_DISPLAY_SIZE = 100; + String FAILED_MESSAGE = " failed (insert=%d update=%d delete=%d)"; String STRUCTURE_NOT_EQUALS = "table structure is not equals , please check the database sync !"; String TABLE_NOT_EXISTS = "table [%s] , only exist in %s !"; diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/report/SliceCheckResultManager.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/report/SliceCheckResultManager.java index c7cb97387c45f73c8a3805e7fc8c8c9df6477e82..61c761d8132e9171b1b8ff0a3a2ca1e2520614b4 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/report/SliceCheckResultManager.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/report/SliceCheckResultManager.java @@ -17,6 +17,7 @@ package org.opengauss.datachecker.check.modules.report; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; + import org.apache.commons.collections4.CollectionUtils; import org.apache.logging.log4j.Logger; import org.opengauss.datachecker.check.client.FeignClientService; @@ -46,6 +47,7 @@ import org.springframework.stereotype.Service; import javax.annotation.Resource; import javax.validation.constraints.NotEmpty; + import java.time.Duration; import java.time.LocalDateTime; import java.util.LinkedList; @@ -73,7 +75,7 @@ public class SliceCheckResultManager { private static final String SUMMARY_LOG_NAME = "summary.log"; private static final String SUCCESS_LOG_NAME = "success.log"; private static final String REPAIR_LOG_TEMPLATE = "repair_%s_%s_%s.txt"; - private static final int MAX_DISPLAY_SIZE = 200; + private static final int MAX_DISPLAY_SIZE = CheckResultConstants.MAX_DISPLAY_SIZE; private final Map tableSliceCountMap = new ConcurrentHashMap<>(); private final Map> checkResult = new ConcurrentHashMap<>(); @@ -83,7 +85,7 @@ public class SliceCheckResultManager { private FeignClientService feignClient; private int successTableCount = 0; private int failedTableCount = 0; - private int rowCount = 0; + private long rowCount = 0L; private boolean isCsvMode = false; private boolean ogCompatibility = false; private boolean hasInitSliceResultEnvironment = true; @@ -92,7 +94,7 @@ public class SliceCheckResultManager { /** * add slice check result * - * @param slice slice + * @param slice slice * @param result result */ public void addResult(SliceVo slice, CheckDiffResult result) { @@ -127,7 +129,7 @@ public class SliceCheckResultManager { /** * add table check result * - * @param table table name + * @param table table name * @param checkDiffResult check result */ public void addResult(String table, CheckDiffResult checkDiffResult) { @@ -140,14 +142,13 @@ public class SliceCheckResultManager { completedSliceSize.set(listValue.size()); return listValue; }); - boolean immediatelyRenameCheckSuccessFile = - Objects.equals(checkDiffResult.getResult(), CheckResultConstants.RESULT_SUCCESS); - notifyCsvShardingCompleted(checkDiffResult, immediatelyRenameCheckSuccessFile); + // is Immediately Rename Check Success File + boolean isImmediately = Objects.equals(checkDiffResult.getResult(), CheckResultConstants.RESULT_SUCCESS); + notifyCsvShardingCompleted(checkDiffResult, isImmediately); if (completedSliceSize.get() == tableSliceCountMap.get(table)) { List results = checkResult.get(table); Map> resultMap; - resultMap = results.stream() - .collect(Collectors.groupingBy(CheckDiffResult::getResult)); + resultMap = results.stream().collect(Collectors.groupingBy(CheckDiffResult::getResult)); String checkResultPath = ConfigCache.getCheckResult(); if (resultMap.containsKey(CheckResultConstants.RESULT_FAILED)) { List tableFiledList = resultMap.get(CheckResultConstants.RESULT_FAILED); @@ -179,12 +180,10 @@ public class SliceCheckResultManager { if (CollectionUtils.isEmpty(csvFailedList)) { return; } - csvFailedList.stream() - .filter(CheckCsvFailed::isNotEmpty) - .forEach(csvFailed -> { - FileUtils.writeAppendFile(csvFailedLogPath, JsonObjectUtil.formatSimple(csvFailed) + ","); - FileUtils.writeAppendFile(csvFailedLogPath, System.lineSeparator()); - }); + csvFailedList.stream().filter(CheckCsvFailed::isNotEmpty).forEach(csvFailed -> { + FileUtils.writeAppendFile(csvFailedLogPath, JsonObjectUtil.formatSimple(csvFailed) + ","); + FileUtils.writeAppendFile(csvFailedLogPath, System.lineSeparator()); + }); } private void notifyCsvShardingCompleted(CheckDiffResult checkDiffResult, boolean immediately) { @@ -200,10 +199,9 @@ public class SliceCheckResultManager { private List translateCheckCsvFaileds(List results) { return results.stream() - .map(result -> new CheckCsvFailed().setTable(result.getTable()) - .build(result.getFileName(), result.getKeyInsert(), - result.getKeyUpdate(), result.getKeyDelete())) - .collect(Collectors.toList()); + .map(result -> new CheckCsvFailed().setTable(result.getTable()) + .build(result.getFileName(), result.getKeyInsert(), result.getKeyUpdate(), result.getKeyDelete())) + .collect(Collectors.toList()); } /** @@ -234,28 +232,32 @@ public class SliceCheckResultManager { Set deleteKeySet = fetchDeleteDiffKeys.fetchKey(tableFailedList); Set updateKeySet = fetchUpdateDiffKeys.fetchKey(tableFailedList); failed.setTopic(new String[] {tableFailedList.get(0).getTopic()}) - .setStartTime(fetchMinStartTime(tableFailedList)) - .setEndTime(fetchMaxEndTime(tableFailedList)) - .setKeyInsertSet(getKeyList(insertKeySet, hasMore, "insert key has more;")) - .setKeyDeleteSet(getKeyList(deleteKeySet, hasMore, "delete key has more;")) - .setKeyUpdateSet(getKeyList(updateKeySet, hasMore, "update key has more;")); - failed.setDiffCount(failed.getKeyInsertSize() + failed.getKeyUpdateSize() + failed.getKeyDeleteSize()); - String message = String.format(FAILED_MESSAGE, failed.getKeyInsertSize(), failed.getKeyUpdateSize(), - failed.getKeyDeleteSize()) + resultCommon.getError(); + .setStartTime(fetchMinStartTime(tableFailedList)) + .setEndTime(fetchMaxEndTime(tableFailedList)) + .setInsertTotal(fetchTotal(tableFailedList, CheckDiffResult::getInsertTotal)) + .setUpdateTotal(fetchTotal(tableFailedList, CheckDiffResult::getUpdateTotal)) + .setDeleteTotal(fetchTotal(tableFailedList, CheckDiffResult::getDeleteTotal)) + .setKeyInsertSet(getKeyList(insertKeySet, hasMore, "insert key has more;")) + .setKeyDeleteSet(getKeyList(deleteKeySet, hasMore, "delete key has more;")) + .setKeyUpdateSet(getKeyList(updateKeySet, hasMore, "update key has more;")); + long diffSum = tableFailedList.stream().peek(msg -> { + log.warn("table slice failed info {} diffSum: {}", msg.getMessage(), msg.getTotalRepair()); + }).mapToLong(CheckDiffResult::getTotalRepair).sum(); + log.warn("result table {} diffSum: {}", resultCommon.getTable(), diffSum); + failed.setDiffCount(diffSum); + String message = + String.format(FAILED_MESSAGE, failed.getInsertTotal(), failed.getUpdateTotal(), failed.getDeleteTotal()) + + resultCommon.getError(); if (resultCommon.isTableStructureEquals()) { failed.setMessage(message); } failed.setHasMore(hasMore.toString()) - .setRowCount(fetchRowCount.fetchCount(tableFailedList) + fetchRowCount.fetchCount(successList)) - .setCost(calcCheckTaskCost(failed.getStartTime(), failed.getEndTime())); + .setRowCount(fetchTotal(tableFailedList, CheckDiffResult::getRowCount) + fetchTotal(successList, + CheckDiffResult::getRowCount)) + .setCost(calcCheckTaskCost(failed.getStartTime(), failed.getEndTime())); return failed; } - private FetchRowCount fetchRowCount = list -> Objects.isNull(list) ? 0 : list.stream() - .mapToLong( - CheckDiffResult::getRowCount) - .sum(); - private FetchDiffKeys fetchInsertDiffKeys = tableFiledList -> { Set diffKey = new TreeSet<>(); tableFiledList.forEach(tableFiled -> { @@ -263,10 +265,7 @@ public class SliceCheckResultManager { diffKey.addAll(tableFiled.getKeyInsertSet()); } if (CollectionUtils.isNotEmpty(tableFiled.getKeyInsert())) { - diffKey.addAll(tableFiled.getKeyInsert() - .stream() - .map(Difference::getKey) - .collect(Collectors.toList())); + diffKey.addAll(tableFiled.getKeyInsert().stream().map(Difference::getKey).collect(Collectors.toList())); } }); return diffKey; @@ -278,10 +277,7 @@ public class SliceCheckResultManager { diffKey.addAll(tableFiled.getKeyDeleteSet()); } if (CollectionUtils.isNotEmpty(tableFiled.getKeyDelete())) { - diffKey.addAll(tableFiled.getKeyDelete() - .stream() - .map(Difference::getKey) - .collect(Collectors.toList())); + diffKey.addAll(tableFiled.getKeyDelete().stream().map(Difference::getKey).collect(Collectors.toList())); } }); return diffKey; @@ -293,10 +289,7 @@ public class SliceCheckResultManager { diffKey.addAll(tableFiled.getKeyUpdateSet()); } if (CollectionUtils.isNotEmpty(tableFiled.getKeyUpdate())) { - diffKey.addAll(tableFiled.getKeyUpdate() - .stream() - .map(Difference::getKey) - .collect(Collectors.toList())); + diffKey.addAll(tableFiled.getKeyUpdate().stream().map(Difference::getKey).collect(Collectors.toList())); } }); return diffKey; @@ -311,15 +304,12 @@ public class SliceCheckResultManager { Set fetchKey(List tableFiledList); } - @FunctionalInterface - protected interface FetchRowCount { - long fetchCount(List tableFiledList); - } - - private long fetchTotalRepair(List tableFiled) { - return tableFiled.stream() - .mapToLong(CheckDiffResult::getTotalRepair) - .sum(); + private long fetchTotal(List tableFiled, + java.util.function.ToLongFunction mapper) { + if (Objects.isNull(tableFiled)) { + return 0L; + } + return tableFiled.stream().mapToLong(mapper).sum(); } private Set getKeyList(Set keySet, StringBuilder hasMore, String message) { @@ -336,10 +326,9 @@ public class SliceCheckResultManager { result.setTopic(new String[] {tableSuccessList.get(0).getTopic()}); result.setStartTime(fetchMinStartTime(tableSuccessList)); result.setEndTime(fetchMaxEndTime(tableSuccessList)); - result.setRowCount(fetchRowCount.fetchCount(tableSuccessList)); + result.setRowCount(fetchTotal(tableSuccessList, CheckDiffResult::getRowCount)); result.setCost(calcCheckTaskCost(result.getStartTime(), result.getEndTime())); - result.setPartition(tableSuccessList.get(0) - .getPartitions()); + result.setPartition(tableSuccessList.get(0).getPartitions()); result.setMessage(fetchMessage(tableSuccessList)); return result; } @@ -348,48 +337,47 @@ public class SliceCheckResultManager { * Calculation and verification time * * @param start start - * @param end end - * @return + * @param end end + * @return Calculation and verification time */ protected long calcCheckTaskCost(LocalDateTime start, LocalDateTime end) { if (Objects.nonNull(start) && Objects.nonNull(end)) { - return Duration.between(start, end) - .toSeconds(); + return Duration.between(start, end).toSeconds(); } return 0; } private LocalDateTime fetchMaxEndTime(@NotEmpty List tableResultList) { - return tableResultList.stream() - .map(CheckDiffResult::getEndTime) - .max(LocalDateTime::compareTo) - .get(); + return tableResultList.stream().map(CheckDiffResult::getEndTime).max(LocalDateTime::compareTo).get(); } private String fetchMessage(@NotEmpty List tableResultList) { - return tableResultList.stream() - .map(CheckDiffResult::getMessage) - .collect(Collectors.toList()) - .toString(); + return tableResultList.stream().map(CheckDiffResult::getMessage).collect(Collectors.toList()).toString(); } private LocalDateTime fetchMinStartTime(@NotEmpty List tableResultList) { - return tableResultList.stream() - .map(CheckDiffResult::getStartTime) - .min(LocalDateTime::compareTo) - .get(); + return tableResultList.stream().map(CheckDiffResult::getStartTime).min(LocalDateTime::compareTo).get(); } private void reduceFailedRepair(String logFilePath, List failedList) { - failedList.forEach(tableFailed -> { - final String repairFile = logFilePath + getRepairFileName(tableFailed); - repairDeleteDiff(repairFile, tableFailed); - repairInsertDiff(repairFile, tableFailed); - repairUpdateDiff(repairFile, tableFailed); - boolean immediatelyRenameCheckFailedFile = - Objects.equals(tableFailed.getResult(), CheckResultConstants.RESULT_FAILED); - notifyCsvShardingCompleted(tableFailed, immediatelyRenameCheckFailedFile); - }); + boolean isCreateRepairSql = ConfigCache.getBooleanValue(ConfigConstants.CREATE_REPAIR_SQL); + if (isCreateRepairSql) { + failedList.forEach(tableFailed -> { + final String repairFile = logFilePath + getRepairFileName(tableFailed); + repairDeleteDiff(repairFile, tableFailed); + repairInsertDiff(repairFile, tableFailed); + repairUpdateDiff(repairFile, tableFailed); + // immediately Rename Check Failed File + boolean isImmediately = Objects.equals(tableFailed.getResult(), CheckResultConstants.RESULT_FAILED); + notifyCsvShardingCompleted(tableFailed, isImmediately); + }); + } else { + if (CollectionUtils.isNotEmpty(failedList)) { + CheckDiffResult tableFailed = failedList.get(0); + final String repairFile = logFilePath + getRepairFileName(tableFailed); + appendLogFile(repairFile, List.of()); + } + } } private void repairUpdateDiff(String repairFile, CheckDiffResult tableFailed) { @@ -398,8 +386,12 @@ public class SliceCheckResultManager { if (CheckResultUtils.isNotEmptyDiff(updateDiffs, keyUpdate)) { RepairEntry update = translateRepairEntry(tableFailed, updateDiffs, keyUpdate); update.setType(DML.REPLACE); - final List updateRepairs = feignClient.buildRepairStatementUpdateDml(Endpoint.SOURCE, update); - appendLogFile(repairFile, updateRepairs); + try { + final List updateRepairs = feignClient.buildRepairStatementUpdateDml(Endpoint.SOURCE, update); + appendLogFile(repairFile, updateRepairs); + } catch (Exception ex) { + log.error("build table {} update repair file {}", tableFailed.getTable(), ex.getMessage()); + } } } @@ -411,9 +403,7 @@ public class SliceCheckResultManager { if (Objects.nonNull(sinkDatabase)) { repairEntry.setSchema(sinkDatabase.getSchema()); } - repairEntry.setDiffSet(diffsSet) - .setOgCompatibility(ogCompatibility) - .setDiffList(keyDifference); + repairEntry.setDiffSet(diffsSet).setOgCompatibility(ogCompatibility).setDiffList(keyDifference); return repairEntry; } @@ -423,8 +413,12 @@ public class SliceCheckResultManager { if (CheckResultUtils.isNotEmptyDiff(insertDiffs, keyInsert)) { RepairEntry insert = translateRepairEntry(tableFailed, insertDiffs, keyInsert); insert.setType(DML.INSERT); - final List insertRepairs = feignClient.buildRepairStatementInsertDml(Endpoint.SOURCE, insert); - appendLogFile(repairFile, insertRepairs); + try { + final List insertRepairs = feignClient.buildRepairStatementInsertDml(Endpoint.SOURCE, insert); + appendLogFile(repairFile, insertRepairs); + } catch (Exception ex) { + log.error("build table {} insert repair file {}", tableFailed.getTable(), ex.getMessage()); + } } } @@ -434,8 +428,12 @@ public class SliceCheckResultManager { if (CheckResultUtils.isNotEmptyDiff(deleteDiffs, keyDelete)) { RepairEntry delete = translateRepairEntry(tableFailed, deleteDiffs, keyDelete); delete.setType(DML.DELETE); - final List deleteRepairs = feignClient.buildRepairStatementDeleteDml(Endpoint.SOURCE, delete); - appendLogFile(repairFile, deleteRepairs); + try { + final List deleteRepairs = feignClient.buildRepairStatementDeleteDml(Endpoint.SOURCE, delete); + appendLogFile(repairFile, deleteRepairs); + } catch (Exception ex) { + log.error("build table {} delete repair file {}", tableFailed.getTable(), ex.getMessage()); + } } } diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckPointSwapRegister.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckPointSwapRegister.java index 212acdead4a4796a1f17385e2099417bf1db9a82..99b01333f515726732b30a7455ada2e4a7919933 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckPointSwapRegister.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckPointSwapRegister.java @@ -16,12 +16,20 @@ package org.opengauss.datachecker.check.service; import com.alibaba.fastjson.JSONObject; + +import cn.hutool.core.bean.BeanUtil; + import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.errors.TimeoutException; import org.apache.logging.log4j.Logger; +import org.opengauss.datachecker.common.config.ConfigCache; +import org.opengauss.datachecker.common.constant.ConfigConstants; +import org.opengauss.datachecker.common.entry.common.CheckPointBean; import org.opengauss.datachecker.common.entry.common.CheckPointData; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.enums.Endpoint; import org.opengauss.datachecker.common.util.LogUtils; import org.opengauss.datachecker.common.util.ThreadUtil; @@ -30,6 +38,8 @@ import org.springframework.kafka.core.KafkaTemplate; import java.time.Duration; import java.util.ArrayList; import java.util.Comparator; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -50,9 +60,12 @@ public class CheckPointSwapRegister { protected static final Map sourcePointCounter = new ConcurrentHashMap<>(); protected static final Map sinkPointCounter = new ConcurrentHashMap<>(); + private static final int MAX_RETRY_TIMES = 100; private static final BlockingQueue CHECK_POINT_QUEUE = new LinkedBlockingQueue<>(); private static final Logger log = LogUtils.getLogger(CheckPointSwapRegister.class); + private final Map> sourceTablePointCache = new ConcurrentHashMap<>(); + private final Map> sinkTablePointCache = new ConcurrentHashMap<>(); private final KafkaServiceManager kafkaServiceManager; private final KafkaTemplate kafkaTemplate; private final ExecutorService checkPointConsumer; @@ -61,6 +74,7 @@ public class CheckPointSwapRegister { private boolean isCompletedSwapTablePoint = false; private boolean isSinkStop; private boolean isSourceStop; + private int maxSliceSize; private volatile String table = null; public CheckPointSwapRegister(KafkaServiceManager kafkaServiceManager, String checkPointTopic) { @@ -69,6 +83,7 @@ public class CheckPointSwapRegister { this.kafkaTemplate = kafkaServiceManager.getKafkaTemplate(); this.checkPointConsumer = ThreadUtil.newSingleThreadExecutor(); this.checkPointSender = ThreadUtil.newSingleThreadExecutor(); + this.maxSliceSize = ConfigCache.getIntValue(ConfigConstants.MAXIMUM_TABLE_SLICE_SIZE); } public void stopMonitor(Endpoint endpoint) { @@ -89,10 +104,9 @@ public class CheckPointSwapRegister { public void registerCheckPoint() { checkPointSender.submit(() -> { - int deliveredCount = 0; CheckPointData calculateCheckPoint = null; - List sourcePoints; - List sinkPoints; + List sourcePoints; + List sinkPoints; while (!isCompletedSwapTablePoint) { synchronized (this) { try { @@ -103,23 +117,21 @@ public class CheckPointSwapRegister { LogUtils.info(log, "start calculate checkpoint [{}][{}]", table, CHECK_POINT_QUEUE.size()); if (table != null && sourcePointCounter.containsKey(table) && sinkPointCounter.containsKey( table)) { - sourcePoints = sourcePointCounter.get(table) - .getCheckPointList(); - sinkPoints = sinkPointCounter.get(table) - .getCheckPointList(); - calculateCheckPoint = - calculateCheckPoint(table, isCheckPointDigit(table), sourcePoints, sinkPoints); + sourcePoints = sourcePointCounter.get(table).getCheckPointList(); + sinkPoints = sinkPointCounter.get(table).getCheckPointList(); + calculateCheckPoint = calculateCheckPoint(table, isCheckPointDigit(table), sourcePoints, + sinkPoints); calculateCheckPoint.setEndpoint(Endpoint.CHECK); - kafkaTemplate.send(checkPointSwapTopicName, Endpoint.CHECK.getDescription(), - JSONObject.toJSONString(calculateCheckPoint)); - deliveredCount++; - LogUtils.info(log, - "send summarized checkpoint topic[{}]:table[{}]:deliverNum[{}]:checkpoint_size[{}]", - checkPointSwapTopicName, calculateCheckPoint.getTableName(), deliveredCount, - calculateCheckPoint.getCheckPointList() - .size()); + final CheckPointData sendCheckPointData = calculateCheckPoint; + sendCheckPointData.getCheckPointList().forEach(point -> { + CheckPointBean tmpBean = new CheckPointBean(); + BeanUtil.copyProperties(sendCheckPointData, tmpBean); + tmpBean.setCheckPoint(point); + sendMsg(checkPointSwapTopicName, Endpoint.CHECK.getDescription(), tmpBean, 0); + }); + log.info("send checkpoint data to extract, table: {} size:{}", + sendCheckPointData.getTableName(), sendCheckPointData.getCheckPointList().size()); } - } catch (Exception ex) { log.error("checkPointSender error {}", table, ex); } @@ -128,11 +140,25 @@ public class CheckPointSwapRegister { }); } + private void sendMsg(String topic, String key, CheckPointBean tmpBean, int reTryTimes) { + try { + kafkaTemplate.send(topic, key, JSONObject.toJSONString(tmpBean)); + } catch (TimeoutException ex) { + if (reTryTimes > MAX_RETRY_TIMES) { + log.error("send msg to kafka timeout, topic: {} key: {} reTryTimes: {}", topic, key, reTryTimes); + } + ThreadUtil.sleepLongCircle(++reTryTimes); + sendMsg(topic, key, tmpBean, reTryTimes); + } + } + private boolean isCheckPointDigit(String table) { - return sourcePointCounter.get(table) - .isDigit(); + return sourcePointCounter.get(table).isDigit(); } + /** + * poll checkpoint data from kafka + */ public void pollSwapPoint() { checkPointConsumer.submit(() -> { KafkaConsumer kafkaConsumer = kafkaServiceManager.getKafkaConsumer(true); @@ -141,19 +167,11 @@ public class CheckPointSwapRegister { while (!isCompletedSwapTablePoint) { try { records = kafkaConsumer.poll(Duration.ofSeconds(1)); - if (!records.isEmpty()) { - records.forEach(record -> { - CheckPointData pointData = JSONObject.parseObject(record.value(), CheckPointData.class); - String tableName = pointData.getTableName(); - if (Objects.equals(record.key(), Endpoint.SOURCE.getDescription())) { - sourcePointCounter.put(tableName, pointData); - tryAddCheckQueue(tableName); - } else if (Objects.equals(record.key(), Endpoint.SINK.getDescription())) { - sinkPointCounter.put(tableName, pointData); - tryAddCheckQueue(tableName); - } - }); - } + processRecords(records); + process(sourceTablePointCache, sourcePointCounter, + "add source checkpoint data to queue, table: {} size:{}"); + process(sinkTablePointCache, sinkPointCounter, + "add sink checkpoint data to queue, table: {} size:{}"); } catch (Exception ex) { if (Objects.equals("java.lang.InterruptedException", ex.getMessage())) { LogUtils.warn(log, "kafka consumer stop by Interrupted"); @@ -163,15 +181,78 @@ public class CheckPointSwapRegister { } } LogUtils.warn(log, "close check point swap consumer {} :{}", checkPointSwapTopicName, - kafkaConsumer.groupMetadata() - .groupId()); + kafkaConsumer.groupMetadata().groupId()); kafkaServiceManager.closeConsumer(kafkaConsumer); }); } + private void processRecords(ConsumerRecords records) { + if (!records.isEmpty()) { + records.forEach(record -> { + CheckPointBean pointBean = JSONObject.parseObject(record.value(), CheckPointBean.class); + if (Objects.equals(record.key(), Endpoint.SOURCE.getDescription())) { + if (sourceTablePointCache.containsKey(pointBean.getTableName())) { + sourceTablePointCache.get(pointBean.getTableName()).add(pointBean); + } else { + List list = new LinkedList<>(); + list.add(pointBean); + sourceTablePointCache.put(pointBean.getTableName(), list); + } + } else if (Objects.equals(record.key(), Endpoint.SINK.getDescription())) { + if (sinkTablePointCache.containsKey(pointBean.getTableName())) { + sinkTablePointCache.get(pointBean.getTableName()).add(pointBean); + } else { + List list = new LinkedList<>(); + list.add(pointBean); + sinkTablePointCache.put(pointBean.getTableName(), list); + } + } else { + log.trace("process ignored"); + } + }); + } + } + + private void process(Map> cache, Map pointCounter, + String message) { + Iterator>> iterator = cache.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> next = iterator.next(); + List value = next.getValue(); + CheckPointBean checkPointBean = value.get(0); + CheckPointData checkPointData = new CheckPointData(); + BeanUtil.copyProperties(checkPointBean, checkPointData); + if (checkPointBean.getSize() == 0) { + checkPointData.setCheckPointList(List.of()); + pointCounter.put(next.getKey(), checkPointData); + tryAddCheckQueue(next.getKey()); + iterator.remove(); + } else if (checkPointBean.getSize() == value.size()) { + checkPointData.setCheckPointList( + value.stream().map(CheckPointBean::getCheckPoint).collect(Collectors.toList())); + checkPointData.setSize(value.size()); + pointCounter.put(next.getKey(), checkPointData); + iterator.remove(); + tryAddCheckQueue(next.getKey()); + log.info(message, next.getKey(), value.size()); + } else { + log.trace("process ignored"); + } + } + } + private void tryAddCheckQueue(String tableName) { if (sourcePointCounter.containsKey(tableName) && sinkPointCounter.containsKey(tableName)) { try { + CheckPointData sourcePoint = sourcePointCounter.get(tableName); + CheckPointData sinkPoint = sinkPointCounter.get(tableName); + if (!validateCheckPointColumn(sourcePoint, sinkPoint)) { + if (sourcePoint.getSize() > sinkPoint.getSize()) { + sourcePointCounter.put(tableName, sinkPoint); + } else { + sinkPointCounter.put(tableName, sourcePoint); + } + } CHECK_POINT_QUEUE.put(tableName); } catch (InterruptedException e) { LogUtils.warn(log, "tryAddCheckQueue occur InterruptedException "); @@ -179,6 +260,10 @@ public class CheckPointSwapRegister { } } + private boolean validateCheckPointColumn(CheckPointData sourcePoint, CheckPointData sinkPoint) { + return StringUtils.equals(sourcePoint.getColName(), sinkPoint.getColName()); + } + private void trySubscribe(KafkaConsumer kafkaConsumer) { int subscribeTimes = 1; boolean isSubscribe = false; @@ -200,37 +285,68 @@ public class CheckPointSwapRegister { return isSubscribe; } - private CheckPointData calculateCheckPoint(String tableName, boolean digit, List checkPointList, - List pointList) { - CheckPointData checkPointData = new CheckPointData(); - List unionCheckPoint = new ArrayList<>() {{ + private CheckPointData calculateCheckPoint(String tableName, boolean isDigit, List checkPointList, + List pointList) { + List unionCheckPoint = new ArrayList<>() {{ addAll(checkPointList); addAll(pointList); }}; - List calculatedCheckpoint = listDistinctAndToSorted(digit, unionCheckPoint); - + List calculatedCheckpoint = listDistinctAndToSorted(isDigit, unionCheckPoint); if (!calculatedCheckpoint.isEmpty()) { checkPointList.add(calculatedCheckpoint.get(0)); checkPointList.add(calculatedCheckpoint.get(calculatedCheckpoint.size() - 1)); } + List lstDistinctOrdered = listDistinctAndToSorted(isDigit, checkPointList); + groupListDistinctOrdered(lstDistinctOrdered); + CheckPointData checkPointData = new CheckPointData(); + CheckPointData result = checkPointData.setTableName(tableName) + .setDigit(isDigit) + .setCheckPointList(lstDistinctOrdered); + result.setSize(result.getCheckPointList().size()); + return result; + } - return checkPointData.setTableName(tableName) - .setDigit(digit) - .setCheckPointList(listDistinctAndToSorted(digit, checkPointList)); + private void groupListDistinctOrdered(List lstDistinctOrdered) { + List inPointPairs = new ArrayList<>(); + int index = 0; + for (PointPair point : lstDistinctOrdered) { + if (canAddPointPairByRowCount(inPointPairs, point, maxSliceSize)) { + point.setSliceIdx(index); + inPointPairs.add(point); + } else { + index++; + point.setSliceIdx(index); + inPointPairs.clear(); + inPointPairs.add(point); + } + } + inPointPairs.clear(); + } + + private boolean canAddPointPairByRowCount(List inPointPairs, PointPair point, int maxSliceSize) { + // 如果沒有检查点，则直接返回true，添加 + if (inPointPairs.isEmpty()) { + return true; + } + // 如果检查点数量大于等于50，则直接返回false，不添加 + if (inPointPairs.size() >= 50) { + return false; + } + // 如果检查点数量小于50，则计算总行数，如果总行数小于等于maxSliceSize，则直接返回true，添加 + // 总行数 = 当前集合中所有检查点的行数 + 新增检查点的行数 + long count = inPointPairs.stream().mapToLong(PointPair::getRowCount).sum(); + return (count + point.getRowCount()) <= maxSliceSize; } - private List listDistinctAndToSorted(boolean digit, List checkPointList) { - return checkPointList.stream() - .distinct() - .sorted(checkPointComparator(digit)) - .collect(Collectors.toList()); + private List listDistinctAndToSorted(boolean isDigit, List checkPointList) { + return checkPointList.stream().distinct().sorted(checkPointComparator(isDigit)).collect(Collectors.toList()); } - private Comparator checkPointComparator(boolean digit) { - if (digit) { - return Comparator.comparingLong(o -> Long.parseLong((String) o)); + private Comparator checkPointComparator(boolean isDigit) { + if (isDigit) { + return Comparator.comparingLong(o -> Long.parseLong((String) o.getCheckPoint())); } else { - return Comparator.comparing(o -> (String) o); + return Comparator.comparing(o -> (String) o.getCheckPoint()); } } } diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckTableStructureService.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckTableStructureService.java index 518448dcb2988f4c6436d7f1e7e366563f742489..e3fe83bf7626f4ecfa85acd4a13848abb063bbec 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckTableStructureService.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckTableStructureService.java @@ -109,7 +109,7 @@ public class CheckTableStructureService { private void checkTableStructureChanged(String processNo) { final List checkTableList = endpointMetaDataManager.getCheckTableList(); - checkTableList.parallelStream().forEach(tableName -> { + checkTableList.forEach(tableName -> { final TableMetadata sourceMeta = endpointMetaDataManager.getTableMetadata(Endpoint.SOURCE, tableName); final TableMetadata sinkMeta = endpointMetaDataManager.getTableMetadata(Endpoint.SINK, tableName); checkTableStructureChanged(processNo, tableName, sourceMeta, sinkMeta); @@ -135,6 +135,7 @@ public class CheckTableStructureService { private void checkTableStructureChanged(String processNo, String tableName, TableMetadata sourceMeta, TableMetadata sinkMeta) { final boolean isTableStructureEquals = isTableStructureEquals(sourceMeta, sinkMeta); + log.info("compared the field names in table[{} {}] ", tableName, isTableStructureEquals); if (!isTableStructureEquals) { taskManagerService.refreshTableExtractStatus(tableName, Endpoint.CHECK, -1); LocalDateTime now = LocalDateTime.now(); @@ -191,8 +192,11 @@ public class CheckTableStructureService { if (isTableNotExist(sourceMeta, sinkMeta)) { return false; } - return tableStructureCompare.compare(sourceMeta.getPrimaryMetas(), sinkMeta.getPrimaryMetas()) - && tableStructureCompare.compare(sourceMeta.getColumnsMetas(), sinkMeta.getColumnsMetas()); + String tableName = sourceMeta.getTableName(); + boolean isPrimary = tableStructureCompare.compare(sourceMeta.getPrimaryMetas(), sinkMeta.getPrimaryMetas()); + boolean isColumn = tableStructureCompare.compare(sourceMeta.getColumnsMetas(), sinkMeta.getColumnsMetas()); + log.info("compared the field names in table[{} {} {}] ", tableName, isPrimary, isColumn); + return isPrimary && isColumn; } @FunctionalInterface @@ -206,7 +210,7 @@ public class CheckTableStructureService { * This is because the database itself determines the column case recognition pattern. * * @param source source - * @param sink sink + * @param sink sink * @return Compare Results */ boolean compare(List source, List sink); diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/ConfigManagement.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/ConfigManagement.java index 7c156db6d6c557b4f053e2dcd73d0e68a5c244db..4fcf91b3ec72280b98be7df726b72ab71e1ac654 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/ConfigManagement.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/ConfigManagement.java @@ -64,6 +64,8 @@ public class ConfigManagement { private int autoDeleteTopic; @Value("${data.check.sql_mode_pad_char_to_full_length}") private boolean sqlModePadCharToFullLength; + @Value("${data.check.create-repair-sql}") + private boolean isCreateRepairSql; @Value("${spring.check.maximum-pool-size}") private int maxPoolSize = 10; /** @@ -81,6 +83,7 @@ public class ConfigManagement { ConfigCache.put(ConfigConstants.REST_API_PAGE_SIZE, restApiPageSize); ConfigCache.put(ConfigConstants.AUTO_DELETE_TOPIC, autoDeleteTopic); ConfigCache.put(ConfigConstants.MAXIMUM_POOL_SIZE, maxPoolSize); + ConfigCache.put(ConfigConstants.CREATE_REPAIR_SQL, isCreateRepairSql); initKafka(); } diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/TaskRegisterCenter.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/TaskRegisterCenter.java index 832e8a0e11e9d480f28e92fd01f1e5d9fbc665b6..cdc467211dc16584f58e552aa42488bb25233447 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/TaskRegisterCenter.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/TaskRegisterCenter.java @@ -28,6 +28,7 @@ import org.opengauss.datachecker.common.util.MapUtils; import org.springframework.stereotype.Component; import javax.annotation.Resource; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,6 +65,7 @@ public class TaskRegisterCenter { private SliceCheckEventHandler sliceCheckEventHandler; private Map sourceIgnoreMap = new HashMap<>(); private Map sinkIgnoreMap = new HashMap<>(); + private Map tableSliceChangeMap = new ConcurrentHashMap<>(); /** * register slice info to register center @@ -122,9 +124,11 @@ public class TaskRegisterCenter { if (curStatus == STATUS_UPDATED_ALL) { sliceCheckEventHandler.handle(new SliceCheckEvent(slice, source, sink)); remove(slice); - }else if (curStatus == STATUS_FAILED) { + } else if (curStatus == STATUS_FAILED) { sliceCheckEventHandler.handleFailed(new SliceCheckEvent(slice, source, sink)); remove(slice); + } else { + log.debug("slice data is extracting {}", slice.getName()); } } } finally { @@ -135,28 +139,23 @@ public class TaskRegisterCenter { private void notifyIgnoreSliceCheckHandle(String table) { removeIgnoreTables(table); List> tableSliceList = center.entrySet() - .stream() - .filter(entry -> entry.getValue() - .getTable() - .equals(table)) - .collect(Collectors.toList()); - Optional.of(tableSliceList) - .ifPresent(list -> { - Entry firstSlice = list.get(0); - String sliceName = firstSlice.getValue() - .getName(); - SliceExtend source = MapUtils.get(sliceExtendMap, sliceName, Endpoint.SOURCE); - SliceExtend sink = MapUtils.get(sliceExtendMap, sliceName, Endpoint.SINK); - sliceCheckEventHandler.handleIgnoreTable(firstSlice.getValue(), source, sink); - String csvDataPath = ConfigCache.getCsvData(); - list.forEach(entry -> { - if (FileUtils.renameTo(csvDataPath, entry.getKey())) { - LogUtils.debug(log, "rename csv sharding completed [{}] by {}", entry.getKey(), - "table miss"); - } - remove(entry.getKey()); - }); - }); + .stream() + .filter(entry -> entry.getValue().getTable().equals(table)) + .collect(Collectors.toList()); + Optional.of(tableSliceList).ifPresent(list -> { + Entry firstSlice = list.get(0); + String sliceName = firstSlice.getValue().getName(); + SliceExtend source = MapUtils.get(sliceExtendMap, sliceName, Endpoint.SOURCE); + SliceExtend sink = MapUtils.get(sliceExtendMap, sliceName, Endpoint.SINK); + sliceCheckEventHandler.handleIgnoreTable(firstSlice.getValue(), source, sink); + String csvDataPath = ConfigCache.getCsvData(); + list.forEach(entry -> { + if (FileUtils.renameTo(csvDataPath, entry.getKey())) { + LogUtils.debug(log, "rename csv sharding completed [{}] by {}", entry.getKey(), "table miss"); + } + remove(entry.getKey()); + }); + }); } /** @@ -198,19 +197,30 @@ public class TaskRegisterCenter { // 处理csv场景已经忽略的表 if (!(sourceIgnoreMap.isEmpty() && sinkIgnoreMap.isEmpty())) { sliceTableCounter.entrySet() - .stream() - .filter(tableEntry -> tableEntry.getValue() > 0 && ( - sourceIgnoreMap.containsKey(tableEntry.getKey()) || sinkIgnoreMap.containsKey( - tableEntry.getKey()))) - .forEach(ignoreTable -> { - notifyIgnoreSliceCheckHandle(ignoreTable.getKey()); - LogUtils.warn(log, "ignore table {} ===add ignore table to result===", - ignoreTable.getKey()); - }); + .stream() + .filter(tableEntry -> tableEntry.getValue() > 0 && (sourceIgnoreMap.containsKey(tableEntry.getKey()) + || sinkIgnoreMap.containsKey(tableEntry.getKey()))) + .forEach(ignoreTable -> { + notifyIgnoreSliceCheckHandle(ignoreTable.getKey()); + LogUtils.warn(log, "ignore table {} ===add ignore table to result===", ignoreTable.getKey()); + }); } - return sliceTableCounter.values() - .stream() - .allMatch(count -> count == 0) && sliceTableCounter.size() == tableCount; + sliceTableCounter.entrySet().stream().filter(tableEntry -> tableEntry.getValue() > 0).forEach((entry) -> { + if (tableSliceChangeMap.containsKey(entry.getKey())) { + if (!Objects.equals(tableSliceChangeMap.get(entry.getKey()), entry.getValue())) { + tableSliceChangeMap.put(entry.getKey(), entry.getValue()); + log.info("check complete release table {} slice {}", entry.getKey(), entry.getValue()); + } + if (entry.getValue() == 0) { + tableSliceChangeMap.remove(entry.getKey()); + } + } else { + tableSliceChangeMap.put(entry.getKey(), entry.getValue()); + log.info("check complete release table {} slice {}", entry.getKey(), entry.getValue()); + } + }); + return sliceTableCounter.values().stream().allMatch(count -> count == 0) + && sliceTableCounter.size() == tableCount; } /** @@ -241,8 +251,8 @@ public class TaskRegisterCenter { * addCheckIgnoreTable * * @param endpoint endpoint - * @param table table - * @param reason reason + * @param table table + * @param reason reason */ public void addCheckIgnoreTable(Endpoint endpoint, String table, String reason) { if (Objects.equals(endpoint, Endpoint.SOURCE)) { diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckEvent.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckEvent.java index 47fb55625a8aaef6c8a9d8835d73638b51b17d98..f1fe2fadf7b3beb60cc97e9cfa8c0b64e88bb76b 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckEvent.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckEvent.java @@ -19,6 +19,8 @@ import lombok.Data; import org.opengauss.datachecker.common.entry.extract.SliceExtend; import org.opengauss.datachecker.common.entry.extract.SliceVo; +import java.util.Objects; + /** * SliceCheckEvent * @@ -35,9 +37,9 @@ public class SliceCheckEvent { /** * slice check event * - * @param slice slice + * @param slice slice * @param source source extend of extract - * @param sink sink extend of extract + * @param sink sink extend of extract */ public SliceCheckEvent(SliceVo slice, SliceExtend source, SliceExtend sink) { this.slice = slice; @@ -52,4 +54,10 @@ public class SliceCheckEvent { public String getCheckName() { return slice.getName(); } + + @Override + public String toString() { + return "checked event : " + slice.getName() + (Objects.nonNull(source) ? source.toString() : " source is null") + + (Objects.nonNull(sink) ? sink.toString() : " sink is null"); + } } diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckEventHandler.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckEventHandler.java index d96d8f870eb0951870cdf1201f66aca3c2dba41e..10331d0e2625890475bdd50f23501826bfab0206 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckEventHandler.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckEventHandler.java @@ -18,12 +18,14 @@ package org.opengauss.datachecker.check.slice; import org.apache.logging.log4j.Logger; import org.opengauss.datachecker.check.modules.check.AbstractCheckDiffResultBuilder.CheckDiffResultBuilder; import org.opengauss.datachecker.check.modules.check.CheckDiffResult; +import org.opengauss.datachecker.check.service.EndpointMetaDataManager; import org.opengauss.datachecker.check.service.TaskRegisterCenter; import org.opengauss.datachecker.common.config.ConfigCache; import org.opengauss.datachecker.common.constant.ConfigConstants; import org.opengauss.datachecker.common.entry.enums.Endpoint; import org.opengauss.datachecker.common.entry.extract.SliceExtend; import org.opengauss.datachecker.common.entry.extract.SliceVo; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.opengauss.datachecker.common.service.DynamicThreadPoolManager; import org.opengauss.datachecker.common.util.LogUtils; import org.springframework.stereotype.Component; @@ -34,6 +36,8 @@ import java.util.concurrent.ThreadPoolExecutor; import static org.opengauss.datachecker.common.constant.DynamicTpConstant.CHECK_EXECUTOR; +import javax.annotation.Resource; + /** * SliceCheckEventHandler * @@ -45,6 +49,8 @@ import static org.opengauss.datachecker.common.constant.DynamicTpConstant.CHECK_ public class SliceCheckEventHandler { private static final Logger log = LogUtils.getLogger(SliceCheckEventHandler.class); + @Resource + private EndpointMetaDataManager metaDataManager; private SliceCheckContext sliceCheckContext; private TaskRegisterCenter registerCenter; private ThreadPoolExecutor executor; @@ -67,22 +73,38 @@ public class SliceCheckEventHandler { * @param checkEvent check event */ public void handle(SliceCheckEvent checkEvent) { - if (checkTableStructure(checkEvent)) { - LogUtils.debug(log, "slice check event {} is dispatched, and checked level=[isTableLevel={}]", - checkEvent.getCheckName(), checkEvent.isTableLevel()); + String tableName = checkEvent.getSlice().getTable(); + boolean isTableHasRows = checkTableHasRows(tableName); + if (!isTableHasRows) { + handleTableEmpty(checkEvent); + registerCenter.refreshCheckedTableCompleted(tableName); + } else if (checkTableStructure(checkEvent)) { + LogUtils.info(log, "slice event {} is dispatched, and checked level=[isTableLevel={} isTableEmpty={}]", + checkEvent.getCheckName(), checkEvent.isTableLevel(), isTableHasRows); if (checkEvent.isTableLevel()) { executor.submit(new TableCheckWorker(checkEvent, sliceCheckContext)); } else { executor.submit(new SliceCheckWorker(checkEvent, sliceCheckContext, registerCenter)); } } else { - LogUtils.info(log, "slice check event , table structure diff [{}][{} : {}]", checkEvent.getCheckName(), - checkEvent.getSource().getTableHash(), checkEvent.getSink().getTableHash()); + LogUtils.info(log, "slice check event , table structure diff [{}]", checkEvent.toString()); handleTableStructureDiff(checkEvent); - registerCenter.refreshCheckedTableCompleted(checkEvent.getSlice().getTable()); + registerCenter.refreshCheckedTableCompleted(tableName); } } + private void handleTableEmpty(SliceCheckEvent checkEvent) { + sliceCheckContext.refreshSliceCheckProgress(checkEvent.getSlice(), 0); + CheckDiffResult result = buildSliceSuccessResult(checkEvent.getSlice(), 0, true); + sliceCheckContext.addCheckResult(checkEvent.getSlice(), result); + } + + private boolean checkTableHasRows(String tableName) { + TableMetadata sourceMeta = metaDataManager.getTableMetadata(Endpoint.SOURCE, tableName); + TableMetadata sinkMeta = metaDataManager.getTableMetadata(Endpoint.SINK, tableName); + return sourceMeta.isExistTableRows() && sinkMeta.isExistTableRows(); + } + /** * 添加校验失败分片事件处理流程 * @@ -133,10 +155,29 @@ public class SliceCheckEventHandler { return builder.build(); } + private CheckDiffResult buildSliceSuccessResult(SliceVo slice, int count, boolean isTableStructure) { + CheckDiffResultBuilder builder = CheckDiffResultBuilder.builder(); + builder.checkMode(ConfigCache.getCheckMode()) + .process(ConfigCache.getValue(ConfigConstants.PROCESS_NO)) + .schema(slice.getSchema()) + .table(slice.getTable()) + .sno(slice.getNo()) + .startTime(LocalDateTime.now()) + .endTime(LocalDateTime.now()) + .isTableStructureEquals(isTableStructure) + .isExistTableMiss(false, null) + .rowCount(count); + return builder.build(); + } + private boolean checkTableStructure(SliceCheckEvent checkEvent) { SliceExtend source = checkEvent.getSource(); SliceExtend sink = checkEvent.getSink(); - return source.getTableHash() == sink.getTableHash(); + if (Objects.nonNull(source) && Objects.nonNull(sink)) { + return source.getTableHash() == sink.getTableHash(); + } else { + return false; + } } /** diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckWorker.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckWorker.java index 26b42af02527f2a60a46d198ddf7e91fc8f77003..86dbfbbf210436d50d1d82df01d4e70cd5479aac 100644 --- a/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckWorker.java +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/slice/SliceCheckWorker.java @@ -27,6 +27,7 @@ import org.opengauss.datachecker.check.modules.bucket.BuilderBucketHandler; import org.opengauss.datachecker.check.modules.bucket.SliceTuple; import org.opengauss.datachecker.check.modules.check.AbstractCheckDiffResultBuilder.CheckDiffResultBuilder; import org.opengauss.datachecker.check.modules.check.CheckDiffResult; +import org.opengauss.datachecker.check.modules.check.CheckResultConstants; import org.opengauss.datachecker.check.modules.check.KafkaConsumerHandler; import org.opengauss.datachecker.check.modules.merkle.MerkleTree; import org.opengauss.datachecker.check.modules.merkle.MerkleTree.Node; @@ -54,6 +55,7 @@ import org.springframework.lang.NonNull; import java.time.LocalDateTime; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; import java.util.List; @@ -63,6 +65,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; /** * SliceCheckWorker @@ -191,6 +194,9 @@ public class SliceCheckWorker implements Runnable { private void checkResult(String resultMsg) { CheckDiffResultBuilder builder = CheckDiffResultBuilder.builder(); + int updateTotal = Objects.nonNull(difference.getDiffering()) ? difference.getDiffering().size() : 0; + int insertTotal = Objects.nonNull(difference.getOnlyOnLeft()) ? difference.getOnlyOnLeft().size() : 0; + int deleteTotal = Objects.nonNull(difference.getOnlyOnRight()) ? difference.getOnlyOnRight().size() : 0; builder.process(ConfigCache.getValue(ConfigConstants.PROCESS_NO)) .table(slice.getTable()) .sno(slice.getNo()) @@ -205,14 +211,23 @@ public class SliceCheckWorker implements Runnable { .endTime(LocalDateTime.now()) .isExistTableMiss(false, null) .rowCount((int) sliceRowCount) - .errorRate(20) + .updateTotal(updateTotal) + .insertTotal(insertTotal) + .deleteTotal(deleteTotal) .checkMode(ConfigCache.getValue(ConfigConstants.CHECK_MODE, CheckMode.class)) - .keyDiff(difference.getOnlyOnLeft(), difference.getDiffering(), difference.getOnlyOnRight()); + .keyDiff(limit(difference.getOnlyOnLeft()), limit(difference.getDiffering()), + limit(difference.getOnlyOnRight())); CheckDiffResult result = builder.build(); LogUtils.debug(LOGGER, "result {}", result); checkContext.addCheckResult(slice, result); } + private List limit(List differences) { + return Objects.nonNull(differences) + ? differences.stream().limit(CheckResultConstants.MAX_DISPLAY_SIZE).collect(Collectors.toList()) + : Collections.emptyList(); + } + private String getConcatTableTopics() { return topic.toTopicString(); } @@ -249,10 +264,6 @@ public class SliceCheckWorker implements Runnable { List entriesOnlyOnLeft = collectorDeleteOrInsert(bucketDifference.entriesOnlyOnLeft()); List entriesOnlyOnRight = collectorDeleteOrInsert(bucketDifference.entriesOnlyOnRight()); List differing = collectorUpdate(bucketDifference.entriesDiffering()); - LogUtils.debug(LOGGER, "diff slice {} insert {}", slice.getName(), bucketDifference.entriesOnlyOnLeft().size()); - LogUtils.debug(LOGGER, "diff slice {} delete {}", slice.getName(), - bucketDifference.entriesOnlyOnRight().size()); - LogUtils.debug(LOGGER, "diff slice {} update {}", slice.getName(), bucketDifference.entriesDiffering().size()); return DifferencePair.of(entriesOnlyOnLeft, entriesOnlyOnRight, differing); } diff --git a/datachecker-check/src/main/resources/application.yml b/datachecker-check/src/main/resources/application.yml index a01ea97ee834a7f241140ad4d6c9c110eec7dd48..3d9eb432b609173d9a8c99c2a136c8ad78e95f6b 100644 --- a/datachecker-check/src/main/resources/application.yml +++ b/datachecker-check/src/main/resources/application.yml @@ -72,6 +72,7 @@ data: check-with-sync-extracting: true retry-fetch-record-times: 5 error-rate: 30 + create-repair-sql: false max-retry-times: 1000 retry-interval-times: 2000 # auto-delete-topic : Configure whether to automatically delete topic. diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/config/ConfigCache.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/config/ConfigCache.java index cc1d695a2d8b9fd2a48492b9bdca43fdc9ea7b3b..5659de94ca5263b95bf1e0a73c33185c46dd137b 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/config/ConfigCache.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/config/ConfigCache.java @@ -141,8 +141,8 @@ public class ConfigCache { * @param key config key * @return config value */ - public static Boolean getBooleanValue(String key) { - return getValue(key, Boolean.class); + public static boolean getBooleanValue(String key) { + return Boolean.TRUE.equals(getValue(key, Boolean.class)); } /** diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/ConfigConstants.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/ConfigConstants.java index 2dd0325945c0e276d22e505b9af21649196229c1..1c475233ef50a88817acb7a8924e29b5c8e3e9f8 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/ConfigConstants.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/ConfigConstants.java @@ -128,6 +128,11 @@ public interface ConfigConstants { */ String MAXIMUM_TABLE_SLICE_SIZE = "spring.check.maximum-table-slice-size"; + /** + * spring.check.maximum-table-slice-size default value 10000 + */ + int MAXIMUM_TABLE_SLICE_DEFAULT_VALUE = 10000; + /** * spring.check.maximum-topic-size */ @@ -312,4 +317,9 @@ public interface ConfigConstants { * lower_case_table_names */ String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; + + /** + * data.check.create-repair-sql + */ + String CREATE_REPAIR_SQL = "data.check.create-repair-sql"; } diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/CheckPointBean.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/CheckPointBean.java new file mode 100644 index 0000000000000000000000000000000000000000..440c8aeeddeed8e2b23aadbf23dfa37e73ad728e --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/CheckPointBean.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +package org.opengauss.datachecker.common.entry.common; + +import lombok.Data; +import lombok.experimental.Accessors; + +import org.opengauss.datachecker.common.entry.enums.Endpoint; + +/** + * CheckPointBean + * + * @author ：wangchao + * @date ：Created in 2023/11/16 + * @since ：11 + */ +@Data +@Accessors(chain = true) +public class CheckPointBean { + private Endpoint endpoint; + private String tableName; + private String colName; + private boolean isDigit; + + /** + * all of the check point list,that count is size + */ + private int size; + private PointPair checkPoint; + + @Override + public String toString() { + return endpoint + "[" + tableName + "." + colName + "] is digit " + isDigit + ", size=" + size + ", checkPoint=" + + checkPoint; + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/CheckPointData.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/CheckPointData.java index 2bbde1c93d5344ec69eb0aa9daa0eba39bcadcdf..1bd8fe9ac30fa1c2f7d170e45a970a9a70b574d8 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/CheckPointData.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/CheckPointData.java @@ -32,12 +32,13 @@ import java.util.List; public class CheckPointData { private Endpoint endpoint; private String tableName; + private String colName; private boolean isDigit; - private List checkPointList; + private int size; + private List checkPointList; @Override public String toString() { - return "endpoint=" + endpoint + ", tableName=" + tableName + ", isDigit=" + isDigit + ", checkPointList=" - + checkPointList.size(); + return endpoint + " [" + tableName + "." + colName + "] is digit=" + isDigit + ", checkPointSize=" + size; } } diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/PointPair.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/PointPair.java new file mode 100644 index 0000000000000000000000000000000000000000..6327688657f8f348a5465f99fe86d8311359b296 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/common/PointPair.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +package org.opengauss.datachecker.common.entry.common; + +import lombok.Data; + +/** + * PointPair + * + * @author ：wangchao + * @date ：Created in 2023/11/16 + * @since ：11 + */ +@Data +public class PointPair { + private Object checkPoint; + + /** + * current checkpoint of table ,contains how manny rows + */ + private long rowCount; + private int sliceIdx; + + public PointPair(Object checkPoint, long rowCount) { + this(checkPoint, rowCount, 0); + } + + public PointPair(Object checkPoint, long rowCount, int sliceIdx) { + this.checkPoint = checkPoint; + this.rowCount = rowCount; + this.sliceIdx = sliceIdx; + } + + @Override + public String toString() { + return "[ " + checkPoint + " , " + rowCount + " idx: " + sliceIdx + " ]"; + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/BaseSlice.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/BaseSlice.java index e09bb5c01e7a6c7efbc756cffdab375433938456..d19f2082ba67a56c0d0f115915bff865a68f6975 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/BaseSlice.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/BaseSlice.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2022 Huawei Technologies Co.,Ltd. + * Copyright (c) 2022-2022 Huawei Technologies Co.,Ltd. All rights reserved. * * openGauss is licensed under Mulan PSL v2. * You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -19,6 +19,8 @@ import lombok.Data; import org.opengauss.datachecker.common.entry.enums.Endpoint; import org.opengauss.datachecker.common.entry.enums.SliceLogType; +import java.util.List; + /** * @author ：wangchao * @date ：Created in 2023/8/2 @@ -48,6 +50,8 @@ public class BaseSlice { * slice of index end value */ private String endIdx; + private List inIds; + private long rowCountOfInIds; /** * number of records in the current slice obtaining table diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractConfig.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractConfig.java index f20366b83bbdc53610d1f7503c07b5b8c18c24ae..e370657bc8a7fdfb6ab07a3314d73998c939a6a8 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractConfig.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractConfig.java @@ -30,6 +30,7 @@ import org.opengauss.datachecker.common.entry.enums.DataLoad; @Accessors(chain = true) public class ExtractConfig { private boolean isDebeziumEnable = false; + private int maxSliceSize = 0; private Database database; private DataLoad dataLoadMode; } diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SliceVo.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SliceVo.java index 9392c3c322b32eaf0d95a27e93ef81a6afab0726..60cdb2e10229f6d3cae08fcce43a8001184fc09d 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SliceVo.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SliceVo.java @@ -16,6 +16,7 @@ package org.opengauss.datachecker.common.entry.extract; import com.alibaba.fastjson.annotation.JSONType; + import lombok.Data; import lombok.EqualsAndHashCode; @@ -37,6 +38,7 @@ public class SliceVo extends BaseSlice { * sink update slice status 2 */ private int status = 0; + private boolean isExistTableRows; /** * table metadata hash value @@ -77,6 +79,9 @@ public class SliceVo extends BaseSlice { if (super.getTotal() == 1) { return super.getName() + " total=" + super.getTotal() + " no=" + super.getNo() + ", [ fetch full ]"; } + if (super.getInIds() != null && !super.getInIds().isEmpty()) { + return super.getName() + " total=" + super.getTotal() + " no=" + super.getNo() + ", " + super.getInIds(); + } return super.getName() + " total=" + super.getTotal() + " no=" + super.getNo() + ", [" + super.getBeginIdx() + " , " + super.getEndIdx() + " ]"; } diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadata.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadata.java index 7283fc966e2605bf41070debc3f79d976c45d0a1..93a5b23b74971a62d36140980977718e0ce8985e 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadata.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadata.java @@ -57,6 +57,7 @@ public class TableMetadata { * If config count_row this will be true. */ private boolean realRows; + private boolean isExistTableRows; private long avgRowLength; @@ -67,6 +68,7 @@ public class TableMetadata { * Primary key column properties */ private List primaryMetas; + private ColumnsMetaData sliceColumn; /** * Table column properties @@ -88,6 +90,18 @@ public class TableMetadata { return primaryMetas.get(0).isAutoIncrementColumn(); } + /** + * 增加联合主键选择机制 + * + * @return ColumnsMetaData + */ + public ColumnsMetaData getSliceColumn() { + if (isSinglePrimary()) { + sliceColumn = primaryMetas.get(0); + } + return sliceColumn; + } + /** * 当前是否是单一主键表 * diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/report/CheckFailed.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/report/CheckFailed.java index c972f1a736eee910f271444b66a0fe273ce850ce..b281771dd1f3d4b68bc48fe315e6b4c7d1f0e15d 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/report/CheckFailed.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/report/CheckFailed.java @@ -20,7 +20,6 @@ import lombok.Data; import lombok.experimental.Accessors; import java.time.LocalDateTime; -import java.util.Objects; import java.util.Set; /** @@ -30,9 +29,10 @@ import java.util.Set; */ @Data @Accessors(chain = true) -@JSONType( - orders = {"process", "schema", "table", "fileName", "topic", "partition", "beginOffset", "rowCount", "diffCount", - "cost", "startTime", "endTime", "message", "keyInsertSet", "keyUpdateSet", "keyDeleteSet", "hasMore"}) +@JSONType(orders = { + "process", "schema", "table", "fileName", "beginOffset", "rowCount", "diffCount", "insertTotal", "updateTotal", + "deleteTotal", "cost", "startTime", "endTime", "message", "keyInsertSet", "keyUpdateSet", "keyDeleteSet", "hasMore" +}) public class CheckFailed { private String process; private String schema; @@ -43,6 +43,9 @@ public class CheckFailed { private int partition; private long rowCount; private long diffCount; + private long insertTotal; + private long updateTotal; + private long deleteTotal; private long cost; private String message; private LocalDateTime startTime; @@ -51,16 +54,4 @@ public class CheckFailed { private Set keyUpdateSet; private Set keyDeleteSet; private String hasMore; - - public int getKeyInsertSize() { - return Objects.nonNull(keyInsertSet) ? keyInsertSet.size() : 0; - } - - public int getKeyUpdateSize() { - return Objects.nonNull(keyUpdateSet) ? keyUpdateSet.size() : 0; - } - - public int getKeyDeleteSize() { - return Objects.nonNull(keyDeleteSet) ? keyDeleteSet.size() : 0; - } } diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/service/DynamicThreadPoolManager.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/service/DynamicThreadPoolManager.java index db348eaa17184a8d1163e11cba64e6050a547eaa..fccec260e2970a5589f89a9557cb19a0e1c0f307 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/service/DynamicThreadPoolManager.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/service/DynamicThreadPoolManager.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; + import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -95,7 +96,7 @@ public class DynamicThreadPoolManager { /** * get Extend Free Thread Pool Executor * - * @param topicSize topicSize + * @param topicSize topicSize * @param extendMaxPoolSize extendMaxPoolSize * @return ExecutorService */ @@ -105,9 +106,9 @@ public class DynamicThreadPoolManager { return EXECUTOR_SERVICE_CACHE.get(freeDtpList.get(0)); } long count = EXECUTOR_SERVICE_CACHE.keySet() - .stream() - .filter(dtpName -> dtpName.startsWith(DynamicTpConstant.EXTEND_EXECUTOR)) - .count(); + .stream() + .filter(dtpName -> dtpName.startsWith(DynamicTpConstant.EXTEND_EXECUTOR)) + .count(); if (count < topicSize) { return buildExtendDtpExecutor(DynamicTpConstant.EXTEND_EXECUTOR + (count + 1), extendMaxPoolSize); } else { @@ -119,8 +120,7 @@ public class DynamicThreadPoolManager { private List getFreeDtpList() { List freeDtpList = new LinkedList<>(); EXECUTOR_SERVICE_CACHE.forEach((dtpName, dtpPool) -> { - if (dtpName.startsWith(DynamicTpConstant.EXTEND_EXECUTOR) && dtpPool.getQueue() - .isEmpty() + if (dtpName.startsWith(DynamicTpConstant.EXTEND_EXECUTOR) && dtpPool.getQueue().isEmpty() && dtpPool.getActiveCount() == 0) { freeDtpList.add(dtpName); } @@ -129,18 +129,29 @@ public class DynamicThreadPoolManager { } private ThreadPoolExecutor buildExtendDtpExecutor(String extendDtpName, int extendMaxPoolSize) { - dynamicThreadPool.buildExtendDtpExecutor(EXECUTOR_SERVICE_CACHE, extendDtpName, corePoolSize, extendMaxPoolSize); + dynamicThreadPool.buildExtendDtpExecutor(EXECUTOR_SERVICE_CACHE, extendDtpName, extendMaxPoolSize, + extendMaxPoolSize); return EXECUTOR_SERVICE_CACHE.get(extendDtpName); } public boolean allExecutorFree() { List freeDtpList = new LinkedList<>(); EXECUTOR_SERVICE_CACHE.forEach((dtpName, dtpPool) -> { - if (dtpPool.getQueue() - .isEmpty() && dtpPool.getActiveCount() == 0) { + if (dtpPool.getQueue().isEmpty() && dtpPool.getActiveCount() == 0) { freeDtpList.add(dtpName); } }); return EXECUTOR_SERVICE_CACHE.size() == freeDtpList.size(); } + + /** + * check executor is busy + * + * @param extendExecutor executor + * @return boolean + */ + public boolean isExecutorBusy(ExecutorService extendExecutor) { + ThreadPoolExecutor pool = (ThreadPoolExecutor) extendExecutor; + return pool.getQueue().size() > pool.getCorePoolSize() * 3; + } } diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/thread/ThreadPoolFactory.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/thread/ThreadPoolFactory.java index 30695771934bfd82c916e8ed5a6186074b2bad10..15746f46721f196a5d7f2aa75b5912a2da91d032 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/thread/ThreadPoolFactory.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/thread/ThreadPoolFactory.java @@ -42,7 +42,7 @@ public class ThreadPoolFactory { private static final double CPU_TIME = 1.0d; private static final double POOL_QUEUE_EXPANSION_RATIO = 1.2d; private static final double CORE_POOL_SIZE_RATIO = 2.0d; - private static final int DEFAULT_QUEUE_SIZE = 1000; + private static final int DEFAULT_QUEUE_SIZE = Integer.MAX_VALUE; /** * Initialize the extract service thread pool diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ThreadUtil.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ThreadUtil.java index 98d2b4aaf0c3432057f15abff8372bcab1d6fea5..078365c408528c5dc44e48d6fc76203800064385 100644 --- a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ThreadUtil.java +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ThreadUtil.java @@ -61,6 +61,19 @@ public class ThreadUtil { } } + /** + * sleep circle,max sleep time is 100 -1000 mill seconds + * + * @param times current circle times + */ + public static void sleepMillsCircle(int times) { + try { + TimeUnit.MILLISECONDS.sleep((times / 10 + 1) * 100L); + } catch (InterruptedException ie) { + LogUtils.warn(log, "thread sleep interrupted exception "); + } + } + /** * sleep circle,max sleep time is 5 seconds (sleep 1-5 sec) * @@ -74,6 +87,19 @@ public class ThreadUtil { } } + /** + * sleep circle,max sleep time is 5 seconds (sleep 1-60 sec) + * + * @param times current circle times + */ + public static void sleepLongCircle(int times) { + try { + TimeUnit.SECONDS.sleep(times / 60 + 1); + } catch (InterruptedException ie) { + LogUtils.warn(log, "thread sleep interrupted exception "); + } + } + /** * The current thread sleeps for 10 - 500 milliseconds */ @@ -118,7 +144,7 @@ public class ThreadUtil { * * @return thread pool */ - @SuppressWarnings( {"all"} ) + @SuppressWarnings( {"all"}) public static ExecutorService newSingleThreadExecutor() { return Executors.newFixedThreadPool(1, Executors.defaultThreadFactory()); } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/TableCheckPointCache.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/TableCheckPointCache.java index c7a505949d6bd7d4574cc8da6d76a57b7defc656..9e0d690ad52060ea078fb7954138a8b1659d2013 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/TableCheckPointCache.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/TableCheckPointCache.java @@ -15,13 +15,10 @@ package org.opengauss.datachecker.extract.cache; -import org.apache.logging.log4j.Logger; -import org.opengauss.datachecker.common.util.LogUtils; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.springframework.lang.NonNull; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -35,39 +32,63 @@ import java.util.concurrent.ConcurrentHashMap; */ @Component public class TableCheckPointCache { - private static final Logger log = LogUtils.getLogger(); - private static final Map> TABLE_CHECKPOINT_CACHE = new ConcurrentHashMap<>(); + private static final Map> TABLE_CHECKPOINT_CACHE = new ConcurrentHashMap<>(); + + private int tableSize; /** * Save to the Cache k v * - * @param key Metadata key + * @param key Metadata key * @param value Metadata value of table */ - public void put(@NonNull String key, List value) { - try { - TABLE_CHECKPOINT_CACHE.put(key, value); - } catch (NumberFormatException exception) { - log.error("put in cache exception ", exception); + public void put(@NonNull String key, List value) { + count(key); + TABLE_CHECKPOINT_CACHE.put(key, value); + } + + private void count(String key) { + if (!TABLE_CHECKPOINT_CACHE.containsKey(key)) { + tableSize++; } } /** - * Get all table checkPointList relationship + * remove table check point cache + * + * @param key key + */ + public void remove(String key) { + TABLE_CHECKPOINT_CACHE.remove(key); + } + + /** + * add point in cache * - * @return map table checkPointList map + * @param key key + * @param value value */ - public Map> getAll() { - try { - return TABLE_CHECKPOINT_CACHE; - } catch (NumberFormatException exception) { - log.error("put in cache exception ", exception); + public void add(@NonNull String key, List value) { + count(key); + if (TABLE_CHECKPOINT_CACHE.containsKey(key)) { + TABLE_CHECKPOINT_CACHE.get(key).addAll(value); + } else { + TABLE_CHECKPOINT_CACHE.put(key, value); } - return new HashMap<>(); + } + + /** + * contains key + * + * @param key key + * @return boolean + */ + public boolean contains(String key) { + return TABLE_CHECKPOINT_CACHE.containsKey(key); } public int tableCount() { - return TABLE_CHECKPOINT_CACHE.size(); + return tableSize; } /** @@ -76,12 +97,7 @@ public class TableCheckPointCache { * @param key table name as cached key * @return list the checkPoint list */ - public List get(String key) { - try { - return TABLE_CHECKPOINT_CACHE.get(key); - } catch (NumberFormatException exception) { - log.error("get cache exception", exception); - return new ArrayList<>(); - } + public List get(String key) { + return TABLE_CHECKPOINT_CACHE.get(key); } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java index 54e94514887f51aa8dd5b9189cbf0303b5d50f74..5d221fdfcae011ce8c4ffda9cfbc0d817919165e 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/BaseDataService.java @@ -110,6 +110,7 @@ public class BaseDataService { if (isChecking) { tableNameList.add(meta.getTableName()); } + meta.setExistTableRows(dataAccessService.tableExistsRows(meta.getTableName())); return isChecking; }).collect(Collectors.toList()); } @@ -179,6 +180,7 @@ public class BaseDataService { if (Objects.isNull(tableMetadata)) { return tableMetadata; } + tableMetadata.setExistTableRows(dataAccessService.tableExistsRows(tableName)); updateTableColumnMetaData(tableMetadata, null); LogUtils.debug(log, "query table metadata {} -- {} ", tableName, tableMetadata); MetaDataCache.put(tableName, tableMetadata); @@ -208,6 +210,7 @@ public class BaseDataService { if (CollectionUtils.isNotEmpty(tempPrimaryColumnBeans)) { List primaryColumnNameList = getPrimaryColumnNames(tempPrimaryColumnBeans); for (ColumnsMetaData column : columns) { + column.setSchema(tableMetadata.getSchema()); if (primaryColumnNameList.contains(column.getLowerCaseColumnName())) { column.setColumnKey(ColumnKey.PRI); } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/AbstractDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/AbstractDataAccessService.java index 93b19b87bd7e11fc9da9aab42a3a3be1e4a55f37..dec8187302214260b14d1e1846e49d63a8ce50e9 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/AbstractDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/AbstractDataAccessService.java @@ -25,6 +25,7 @@ import org.opengauss.datachecker.common.config.ConfigCache; import org.opengauss.datachecker.common.constant.ConfigConstants; import org.opengauss.datachecker.common.entry.check.Difference; import org.opengauss.datachecker.common.entry.common.Health; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.enums.LowerCaseTableNames; import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; import org.opengauss.datachecker.common.entry.extract.TableMetadata; @@ -84,11 +85,7 @@ public abstract class AbstractDataAccessService implements DataAccessService { * @return connection */ protected Connection getConnection() { - try { - return druidDataSource.getConnection(); - } catch (SQLException e) { - throw new ExtractDataAccessException(e); - } + return ConnectionMgr.getConnection(); } /** @@ -306,6 +303,25 @@ public abstract class AbstractDataAccessService implements DataAccessService { return list; } + /** + * query union point list + * + * @param connection conn + * @param sql sql + * @return point list + */ + protected List adasQueryUnionPointList(Connection connection, String sql) { + List list = new LinkedList<>(); + try (PreparedStatement ps = connection.prepareStatement(sql); ResultSet resultSet = ps.executeQuery()) { + while (resultSet.next()) { + list.add(new PointPair(resultSet.getString(1), resultSet.getLong(2))); + } + } catch (SQLException esql) { + LogUtils.error(log, "adasQueryPointList error", esql); + } + return list; + } + /** * 查询表数据抽样检查点清单 * diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/CsvDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/CsvDataAccessService.java index 322796bbd00a0004d5ebed5cdd7651482e37eb05..b1d1645f6a9bc69566b2f7c6f9bdcabf12a032be 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/CsvDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/CsvDataAccessService.java @@ -24,6 +24,7 @@ import org.opengauss.datachecker.common.config.ConfigCache; import org.opengauss.datachecker.common.entry.check.Difference; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.common.Health; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.csv.CsvTableColumnMeta; import org.opengauss.datachecker.common.entry.csv.CsvTableMeta; import org.opengauss.datachecker.common.entry.enums.ColumnKey; @@ -250,6 +251,11 @@ public class CsvDataAccessService implements DataAccessService { return 0; } + @Override + public boolean tableExistsRows(String tableName) { + return false; + } + @Override public String min(Connection connection, DataAccessParam param) { return null; @@ -274,4 +280,9 @@ public class CsvDataAccessService implements DataAccessService { public List queryTableUniqueColumns(String tableName) { return null; } + + @Override + public List queryUnionFirstPrimaryCheckPointList(Connection connection, DataAccessParam param) { + return null; + } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/DataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/DataAccessService.java index 6eaef3875e047addeba0a532d739732a8783d80e..2ed6a8a2a31da69ffc27a53beee975c48ddd3a5d 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/DataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/DataAccessService.java @@ -18,6 +18,7 @@ package org.opengauss.datachecker.extract.data.access; import org.opengauss.datachecker.common.entry.check.Difference; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.common.Health; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.enums.LowerCaseTableNames; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; @@ -108,6 +109,14 @@ public interface DataAccessService { */ long rowCount(String tableName); + /** + * query table is empty + * + * @param tableName tableName + * @return true or false + */ + boolean tableExistsRows(String tableName); + /** * query table column min value * @@ -199,4 +208,13 @@ public interface DataAccessService { * @return unique columns */ List queryTableUniqueColumns(String tableName); + + /** + * query table check point list + * + * @param connection connection + * @param param param + * @return point list + */ + List queryUnionFirstPrimaryCheckPointList(Connection connection, DataAccessParam param); } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java index afee7093c45a40b82e9f3cd6c64e27c7e2b5ed2f..7bf6a3d8a089a92f28536fde3ee7c11beb5a523b 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/MysqlDataAccessService.java @@ -17,6 +17,7 @@ package org.opengauss.datachecker.extract.data.access; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.common.Health; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.enums.LowerCaseTableNames; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; @@ -122,6 +123,11 @@ public class MysqlDataAccessService extends AbstractDataAccessService { return mysqlMetaDataMapper.rowCount(properties.getSchema(), tableName); } + @Override + public boolean tableExistsRows(String tableName) { + return mysqlMetaDataMapper.tableExistsRows(properties.getSchema(), tableName); + } + @Override public String min(Connection connection, DataAccessParam param) { String sql = " select min(" + param.getColName() + ") from " + param.getSchema() + "." + param.getName() + ";"; @@ -148,6 +154,13 @@ public class MysqlDataAccessService extends AbstractDataAccessService { return adasQueryPointList(connection, sql); } + @Override + public List queryUnionFirstPrimaryCheckPointList(Connection connection, DataAccessParam param) { + String sqlTmp = "select %s,count(1) from %s.%s group by %s"; + String sql = String.format(sqlTmp, param.getColName(), param.getSchema(), param.getName(), param.getColName()); + return adasQueryUnionPointList(connection, sql); + } + @Override public boolean dasCheckDatabaseNotEmpty() { return mysqlMetaDataMapper.checkDatabaseNotEmpty(properties.getSchema()); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java index 86a0843112937fe99c9c0fa7f7a4283a0c057d40..fb3851f68aeea955d1adef0d70dceb76686f776f 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OpgsDataAccessService.java @@ -19,6 +19,7 @@ import org.opengauss.datachecker.common.config.ConfigCache; import org.opengauss.datachecker.common.constant.ConfigConstants; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.common.Health; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.enums.LowerCaseTableNames; import org.opengauss.datachecker.common.entry.enums.OgCompatibility; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; @@ -168,6 +169,11 @@ public class OpgsDataAccessService extends AbstractDataAccessService { return opgsMetaDataMapper.rowCount(properties.getSchema(), tableName); } + @Override + public boolean tableExistsRows(String tableName) { + return opgsMetaDataMapper.tableExistsRows(properties.getSchema(), tableName); + } + @Override public String min(Connection connection, DataAccessParam param) { String sql = " select min(" + param.getColName() + ") from " + param.getSchema() + "." + param.getName() + ";"; @@ -194,6 +200,13 @@ public class OpgsDataAccessService extends AbstractDataAccessService { return adasQueryPointList(connection, sql); } + @Override + public List queryUnionFirstPrimaryCheckPointList(Connection connection, DataAccessParam param) { + String sqlTmp = "select %s,count(1) from %s.%s group by %s"; + String sql = String.format(sqlTmp, param.getColName(), param.getSchema(), param.getName(), param.getColName()); + return adasQueryUnionPointList(connection, sql); + } + @Override public boolean dasCheckDatabaseNotEmpty() { return opgsMetaDataMapper.checkDatabaseNotEmpty(properties.getSchema()); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OracleDataAccessService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OracleDataAccessService.java index bfaef4565f44cfc4eae433bd9fd544614aa93095..ee515fe07c19e440979f25e09b0252becd1bd64e 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OracleDataAccessService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/access/OracleDataAccessService.java @@ -17,6 +17,7 @@ package org.opengauss.datachecker.extract.data.access; import org.opengauss.datachecker.common.entry.common.DataAccessParam; import org.opengauss.datachecker.common.entry.common.Health; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.enums.LowerCaseTableNames; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; import org.opengauss.datachecker.common.entry.extract.PrimaryColumnBean; @@ -118,6 +119,11 @@ public class OracleDataAccessService extends AbstractDataAccessService { return oracleMetaDataMapper.rowCount(properties.getSchema(), tableName); } + @Override + public boolean tableExistsRows(String tableName) { + return oracleMetaDataMapper.tableExistsRows(properties.getSchema(), tableName); + } + @Override public String min(Connection connection, DataAccessParam param) { return oracleMetaDataMapper.min(param); @@ -138,6 +144,12 @@ public class OracleDataAccessService extends AbstractDataAccessService { return oracleMetaDataMapper.queryPointList(param); } + @Override + public List queryUnionFirstPrimaryCheckPointList(Connection connection, DataAccessParam param) { + // oracle database`table,that is defined by union primary key + return null; + } + @Override public boolean dasCheckDatabaseNotEmpty() { return oracleMetaDataMapper.checkDatabaseNotEmpty(properties.getSchema()); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/MetaDataMapper.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/MetaDataMapper.java index b1774da2c812ab39fff702928ad7d381d92151e0..9d4057213aa88ac506c986fa53fef2dc3f5401dd 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/MetaDataMapper.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/data/mapper/MetaDataMapper.java @@ -144,4 +144,13 @@ public interface MetaDataMapper { * @return boolean */ boolean checkDatabaseNotEmpty(@Param("schema") String schema); + + /** + * check table is not empty + * + * @param schema scheam + * @param tableName table name + * @return is not empty + */ + boolean tableExistsRows(@Param("schema") String schema, @Param("name") String tableName); } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/load/StartLoadRunner.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/load/StartLoadRunner.java index 8d4cff06fb57f05d16f72bfffec37e0670843cf6..bd0825786c83c84f6c6c07d851cfb8e6d3e4550b 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/load/StartLoadRunner.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/load/StartLoadRunner.java @@ -55,9 +55,9 @@ public class StartLoadRunner implements ApplicationRunner { @Override public void run(ApplicationArguments args) { // if extract boot start finished,then running. + resourceManager.initMaxConnectionCount(); configManagement.loadExtractProperties(); memoryManagerService.startMemoryManager(ConfigCache.getBooleanValue(ConfigConstants.MEMORY_MONITOR)); - resourceManager.initMaxConnectionCount(); dynamicThreadPoolManager.dynamicThreadPoolMonitor(); initExtractContextDataSource(); sliceProcessorContext.startSliceStatusFeedbackService(); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/ConnectionMgr.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/ConnectionMgr.java index b51731fa05fac79e03a7006ad4cb3815c05f3656..139ade3129f73167f5ab496d490cd4793250ba4b 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/ConnectionMgr.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/ConnectionMgr.java @@ -23,12 +23,12 @@ import org.opengauss.datachecker.common.util.LogUtils; import org.opengauss.datachecker.common.util.ThreadUtil; import javax.sql.PooledConnection; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -43,7 +43,7 @@ import java.util.concurrent.locks.ReentrantLock; public class ConnectionMgr { private static final Logger log = LogUtils.getLogger(ConnectionMgr.class); private static final Lock LOCK = new ReentrantLock(); - private static final int CREATE_CONNECTION_RETRY_TIMES = 30; + private static final int MAX_RETRY_TIMES = 60; private static String driverClassName = ""; private static String url = ""; @@ -54,9 +54,9 @@ public class ConnectionMgr { /** * 获取JDBC链接 * - * @return Connection + * @return jdbc Connection */ - public static Connection getConnection() { + public static synchronized Connection getConnection() { if (isFirstLoad.get()) { driverClassName = getPropertyValue(ConfigConstants.DRIVER_CLASS_NAME); url = getPropertyValue(ConfigConstants.DS_URL); @@ -73,18 +73,8 @@ public class ConnectionMgr { Connection conn = null; LOCK.lock(); try { - conn = tryToCreateConnection(); int retry = 0; - while (Objects.isNull(conn) && retry < CREATE_CONNECTION_RETRY_TIMES) { - ThreadUtil.sleepMaxHalfSecond(); - conn = tryToCreateConnection(); - retry++; - } - if (Objects.isNull(conn)) { - throw new ExtractDataAccessException("create connection failed " + CREATE_CONNECTION_RETRY_TIMES + " times"); - } - conn.setAutoCommit(false); - LogUtils.debug(log, "Connection succeed !"); + conn = retryToGetConnection(retry); } catch (Exception ignore) { LogUtils.error(log, "create connection failed , [{},{}]:[{}][{}]", username, databasePassport, url, ignore.getMessage()); @@ -94,14 +84,19 @@ public class ConnectionMgr { return conn; } - private static Connection tryToCreateConnection() throws SQLException { - Connection conn = null; + private static Connection retryToGetConnection(int retry) throws SQLException { + Connection conn; try { - ThreadUtil.sleepOneSecond(); conn = DriverManager.getConnection(url, username, databasePassport); - LogUtils.debug(log, "Connection succeed !"); + conn.setAutoCommit(false); } catch (Exception exp) { - LogUtils.error(log, "create connection failed , [{},{}]:[{}]", username, databasePassport, url, exp); + if (retry <= MAX_RETRY_TIMES) { + LogUtils.error(log, "retry to get connection cause by {}", exp.getMessage()); + ThreadUtil.sleepCircle(retry); + conn = retryToGetConnection(++retry); + } else { + throw exp; + } } return conn; } @@ -114,8 +109,8 @@ public class ConnectionMgr { * 关闭数据库链接及 PreparedStatement、ResultSet结果集 * * @param connection connection - * @param ps PreparedStatement - * @param resultSet resultSet + * @param ps PreparedStatement + * @param resultSet resultSet */ public static void close(Connection connection, PreparedStatement ps, ResultSet resultSet) { if (resultSet != null) { diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/JdbcDataOperations.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/JdbcDataOperations.java index 73ba276b9bb8813a064f1bcf290f668e2f9dd24e..66188d597e66ce2e1dbcff076d55969f544d4d96 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/JdbcDataOperations.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/resource/JdbcDataOperations.java @@ -16,6 +16,7 @@ package org.opengauss.datachecker.extract.resource; import com.alibaba.druid.pool.DruidDataSource; + import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.opengauss.datachecker.common.config.ConfigCache; @@ -27,6 +28,7 @@ import org.opengauss.datachecker.common.util.ThreadUtil; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Objects; @@ -97,11 +99,11 @@ public class JdbcDataOperations { * try to get a jdbc connection and close auto commit. * * @param allocMemory allocMemory - * @param dataSource dataSource + * @param dataSource dataSource * @return Connection */ - public synchronized Connection tryConnectionAndClosedAutoCommit(long allocMemory, - DruidDataSource dataSource) throws SQLException { + public synchronized Connection tryConnectionAndClosedAutoCommit(long allocMemory, DruidDataSource dataSource) + throws SQLException { takeConnection(allocMemory); return getConnectionAndClosedAutoCommit(dataSource); } @@ -184,8 +186,20 @@ public class JdbcDataOperations { * @param connection connection */ public synchronized void releaseConnection(Connection connection) { - resourceManager.release(); ConnectionMgr.close(connection, null, null); + resourceManager.release(); + } + + /** + * release connection resource,and close ps & rs + * + * @param connection connect + * @param ps ps + * @param rs rs + */ + public synchronized void releaseConnection(Connection connection, PreparedStatement ps, ResultSet rs) { + ConnectionMgr.close(connection, ps, rs); + resourceManager.release(); } /** diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractServiceImpl.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractServiceImpl.java index fd607fa52b72f8a7686ade1ca41bdaa09667e5a0..39df39e66e21e3a5702ee8c747dd224efcadb626 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractServiceImpl.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractServiceImpl.java @@ -21,24 +21,26 @@ import org.opengauss.datachecker.common.config.ConfigCache; import org.opengauss.datachecker.common.constant.ConfigConstants; import org.opengauss.datachecker.common.constant.Constants; import org.opengauss.datachecker.common.entry.common.CheckPointData; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.enums.SliceStatus; +import org.opengauss.datachecker.common.entry.extract.BaseSlice; import org.opengauss.datachecker.common.entry.extract.Database; import org.opengauss.datachecker.common.entry.extract.ExtractConfig; import org.opengauss.datachecker.common.entry.extract.ExtractTask; import org.opengauss.datachecker.common.entry.extract.PageExtract; import org.opengauss.datachecker.common.entry.extract.RowDataHash; +import org.opengauss.datachecker.common.entry.extract.SliceExtend; import org.opengauss.datachecker.common.entry.extract.SliceVo; import org.opengauss.datachecker.common.entry.extract.SourceDataLog; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.opengauss.datachecker.common.entry.extract.TableMetadataHash; -import org.opengauss.datachecker.common.entry.extract.Topic; import org.opengauss.datachecker.common.exception.ProcessMultipleException; import org.opengauss.datachecker.common.exception.TableNotExistException; import org.opengauss.datachecker.common.exception.TaskNotFoundException; import org.opengauss.datachecker.common.service.DynamicThreadPoolManager; import org.opengauss.datachecker.common.util.LogUtils; import org.opengauss.datachecker.common.util.ThreadUtil; -import org.opengauss.datachecker.common.util.TopicUtil; import org.opengauss.datachecker.extract.cache.MetaDataCache; import org.opengauss.datachecker.extract.cache.TableCheckPointCache; import org.opengauss.datachecker.extract.cache.TableExtractStatusCache; @@ -48,6 +50,7 @@ import org.opengauss.datachecker.extract.config.KafkaConsumerConfig; import org.opengauss.datachecker.extract.data.BaseDataService; import org.opengauss.datachecker.extract.data.access.DataAccessService; import org.opengauss.datachecker.extract.slice.ExtractPointSwapManager; +import org.opengauss.datachecker.extract.slice.SliceProcessorContext; import org.opengauss.datachecker.extract.slice.SliceRegister; import org.opengauss.datachecker.extract.slice.factory.SliceFactory; import org.opengauss.datachecker.extract.task.CheckPoint; @@ -66,6 +69,7 @@ import org.springframework.util.StopWatch; import javax.annotation.Resource; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -74,6 +78,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -136,6 +141,8 @@ public class DataExtractServiceImpl implements DataExtractService { @Resource private KafkaConsumerConfig kafkaConsumerConfig; private ExtractPointSwapManager checkPointManager = null; + @Resource + private SliceProcessorContext sliceProcessorContext; /** * Data extraction service @@ -327,25 +334,53 @@ public class DataExtractServiceImpl implements DataExtractService { tableName); return; } - Endpoint endpoint = extractProperties.getEndpoint(); - while (!tableCheckPointCache.getAll().containsKey(tableName)) { - ThreadUtil.sleepHalfSecond(); + TableMetadata tableMetadata = task.getTableMetadata(); + if (!tableMetadata.isExistTableRows()) { + emptyTableSliceProcessor(tableMetadata); + } else { + Endpoint endpoint = extractProperties.getEndpoint(); + while (!tableCheckPointCache.contains(tableName)) { + ThreadUtil.sleepHalfSecond(); + } + List summarizedCheckPoint = tableCheckPointCache.get(tableName); + LogUtils.debug(log, "table [{}] summarized check-point-list : {}", tableName, + summarizedCheckPoint.size()); + List sliceVoList = buildSliceByTask(summarizedCheckPoint, tableMetadata, endpoint); + LogUtils.info(log, "table [{}] have {} slice to check", tableName, sliceVoList.size()); + addSliceProcessor(sliceVoList); } - List summarizedCheckPoint = tableCheckPointCache.get(tableName); - LogUtils.debug(log, "table [{}] summarized check-point-list : {}", tableName, summarizedCheckPoint); - List sliceVoList = buildSliceByTask(summarizedCheckPoint, task.getTableMetadata(), endpoint); - LogUtils.info(log, "table [{}] have {} slice to check", tableName, sliceVoList.size()); - addSliceProcessor(sliceVoList); + tableCheckPointCache.remove(tableName); } catch (Exception ex) { LogUtils.error(log, "async exec extract tables error {}:{} ", task.getTableName(), ex.getMessage(), ex); } } - private List buildSliceByTask(List summarizedCheckPoint, TableMetadata tableMetadata, + private void emptyTableSliceProcessor(TableMetadata tableMetadata) { + String tableName = tableMetadata.getTableName(); + SliceVo slice = new SliceVo(); + SliceExtend sliceExtend = new SliceExtend(); + slice.setName(sliceTaskNameBuilder(tableName, 0)); + slice.setNo(1); + slice.setTable(tableName); + slice.setTotal(1); + slice.setStatus(SliceStatus.codeOf(ConfigCache.getEndPoint())); + slice.setEndpoint(ConfigCache.getEndPoint()); + slice.setTableHash(tableMetadata.getTableHash()); + slice.setExistTableRows(tableMetadata.isExistTableRows()); + sliceRegister.batchRegister(List.of(slice)); + BeanUtils.copyProperties(slice, sliceExtend); + ThreadUtil.sleepOneSecond(); + sliceProcessorContext.feedbackStatus(sliceExtend); + log.info("add empty table slice task feedback status: {}->{}", tableName, sliceExtend.getStatus()); + } + + private List buildSliceByTask(List summarizedCheckPoint, TableMetadata tableMetadata, Endpoint endpoint) { List sliceVoList; if (noTableSlice(tableMetadata, summarizedCheckPoint)) { sliceVoList = buildSingleSlice(tableMetadata, endpoint); + } else if (tableMetadata.isUnionPrimary()) { + sliceVoList = buildUnionPrimaryTableSlice(summarizedCheckPoint, tableMetadata, endpoint); } else { sliceVoList = buildSlice(summarizedCheckPoint, tableMetadata, endpoint); } @@ -353,40 +388,47 @@ public class DataExtractServiceImpl implements DataExtractService { } private void addSliceProcessor(List sliceVoList) { - SliceFactory sliceFactory = new SliceFactory(baseDataService.getDataSource()); sliceRegister.batchRegister(sliceVoList); + int sliceSize = sliceVoList.size(); + int topicSize = ConfigCache.getIntValue(ConfigConstants.MAXIMUM_TOPIC_SIZE); + int extendMaxPoolSize = ConfigCache.getIntValue(ConfigConstants.EXTEND_MAXIMUM_POOL_SIZE); + ExecutorService executor; if (sliceVoList.size() <= 20) { - ExecutorService executorService = dynamicThreadPoolManager.getExecutor(EXTRACT_EXECUTOR); - LogUtils.debug(log, "table [{}] get executorService success", sliceVoList.get(0).getTable()); - sliceVoList.forEach(sliceVo -> { - executorService.submit(sliceFactory.createSliceProcessor(sliceVo)); - }); + executor = dynamicThreadPoolManager.getExecutor(EXTRACT_EXECUTOR); } else { - int topicSize = ConfigCache.getIntValue(ConfigConstants.MAXIMUM_TOPIC_SIZE); - int extendMaxPoolSize = ConfigCache.getIntValue(ConfigConstants.EXTEND_MAXIMUM_POOL_SIZE); - ExecutorService extendExecutor = dynamicThreadPoolManager.getFreeExecutor(topicSize, extendMaxPoolSize); - LogUtils.debug(log, "table [{}] get extendExecutor success", sliceVoList.get(0).getTable()); - sliceVoList.forEach(sliceVo -> { - extendExecutor.submit(sliceFactory.createSliceProcessor(sliceVo)); - }); + executor = dynamicThreadPoolManager.getFreeExecutor(topicSize, extendMaxPoolSize); } + AtomicInteger lastShardSize = new AtomicInteger(sliceSize); + SliceFactory sliceFactory = new SliceFactory(baseDataService.getDataSource()); + String table = sliceVoList.get(0).getTable(); + AtomicInteger sliceCount = new AtomicInteger(sliceSize); + sliceVoList.forEach(sliceVo -> { + int bussyWait = 0; + while (dynamicThreadPoolManager.isExecutorBusy(executor)) { + // 线程池满了，等待线程池释放线程 + if (lastShardSize.get() != sliceCount.get()) { + lastShardSize.set(sliceCount.get()); + log.warn("executor is busy. table {} has {} shards waiting to be added to queue, total {} shards.", + table, sliceCount.get(), sliceSize); + } + ThreadUtil.sleepMillsCircle(++bussyWait); + } + executor.submit(sliceFactory.createSliceProcessor(sliceVo)); + sliceCount.getAndDecrement(); + log.info("shard is added to executor. table {} has {} shards remaining and a total of " + "{} shards.", + table, sliceCount.get(), sliceVoList.size()); + }); } private List buildSingleSlice(TableMetadata metadata, Endpoint endpoint) { - SliceVo sliceVo = new SliceVo(); + SliceVo sliceVo = buildTmpSlice(metadata, endpoint); sliceVo.setNo(SINGLE_SLICE_NUM); - sliceVo.setTable(metadata.getTableName()); - sliceVo.setSchema(metadata.getSchema()); sliceVo.setName(sliceTaskNameBuilder(metadata.getTableName(), 0)); - sliceVo.setPtnNum(1); - sliceVo.setPtn(0); sliceVo.setTotal(SINGLE_SLICE_NUM); - sliceVo.setEndpoint(endpoint); - sliceVo.setFetchSize(ConfigCache.getIntValue(ConfigConstants.MAXIMUM_TABLE_SLICE_SIZE)); return List.of(sliceVo); } - private boolean noTableSlice(TableMetadata tableMetadata, List summarizedCheckPoint) { + private boolean noTableSlice(TableMetadata tableMetadata, List summarizedCheckPoint) { return summarizedCheckPoint.size() <= 2 || getQueryDop() == 1 || tableMetadata.getConditionLimit() != null; } @@ -394,37 +436,83 @@ public class DataExtractServiceImpl implements DataExtractService { return ConfigCache.getIntValue(ConfigConstants.QUERY_DOP); } - private List buildSlice(List summarizedCheckPoint, TableMetadata metadata, Endpoint endpoint) { + private List buildUnionPrimaryTableSlice(List summarizedCheckPoint, TableMetadata metadata, + Endpoint endpoint) { + SliceVo tmp = buildTmpSlice(metadata, endpoint); + long tableRowCount = summarizedCheckPoint.stream().mapToLong(PointPair::getRowCount).sum(); + if (tableRowCount < tmp.getFetchSize()) { + tmp.setNo(1); + tmp.setTotal(1); + tmp.setRowCountOfInIds(tableRowCount); + tmp.setName(sliceTaskNameBuilder(metadata.getTableName(), 0)); + return List.of(tmp); + } + Map> groupedMap = summarizedCheckPoint.stream() + .collect(Collectors.groupingBy(PointPair::getSliceIdx)); + int totalSliceSize = groupedMap.keySet().size(); + List sliceTaskList = new ArrayList<>(); + for (List groupList : groupedMap.values()) { + SliceVo sliceVo = new SliceVo(); + BeanUtils.copyProperties(tmp, sliceVo); + PointPair pointPair = groupList.get(0); + sliceVo.setName(sliceTaskNameBuilder(metadata.getTableName(), pointPair.getSliceIdx())); + sliceVo.setInIds( + groupList.stream().map(p -> String.valueOf(p.getCheckPoint())).collect(Collectors.toList())); + sliceVo.setNo(pointPair.getSliceIdx() + 1); + sliceVo.setRowCountOfInIds(groupList.stream().mapToLong(PointPair::getRowCount).sum()); + sliceVo.setTotal(totalSliceSize); + sliceTaskList.add(sliceVo); + } + sliceTaskList.sort(Comparator.comparingInt(BaseSlice::getNo)); + log.info("build slice task list success, table {} merge slice {}-> {}", metadata.getTableName(), + summarizedCheckPoint.size(), sliceTaskList.size()); + return sliceTaskList; + } + + private SliceVo buildTmpSlice(TableMetadata metadata, Endpoint endpoint) { + SliceVo tmp = new SliceVo(); + tmp.setTable(metadata.getTableName()); + tmp.setSchema(metadata.getSchema()); + tmp.setFetchSize(ConfigCache.getIntValue(ConfigConstants.MAXIMUM_TABLE_SLICE_SIZE)); + tmp.setPtnNum(1); + tmp.setPtn(0); + tmp.setEndpoint(endpoint); + return tmp; + } + + private List buildSlice(List summarizedCheckPoint, TableMetadata metadata, Endpoint endpoint) { ArrayList sliceTaskList = new ArrayList<>(); - Iterator iterator = summarizedCheckPoint.iterator(); - Object preOffset = iterator.next(); + Iterator iterator = summarizedCheckPoint.iterator(); + PointPair preOffset = iterator.next(); int index = 0; + SliceVo tmp = buildTmpSlice(metadata, endpoint); while (iterator.hasNext()) { - Object offset = iterator.next(); + PointPair point = iterator.next(); + Object offset = point.getCheckPoint(); SliceVo sliceVo = new SliceVo(); - sliceVo.setTable(metadata.getTableName()); - sliceVo.setSchema(metadata.getSchema()); - sliceVo.setFetchSize(ConfigCache.getIntValue(ConfigConstants.MAXIMUM_TABLE_SLICE_SIZE)); + BeanUtils.copyProperties(tmp, sliceVo); sliceVo.setName(sliceTaskNameBuilder(metadata.getTableName(), index)); - sliceVo.setBeginIdx(String.valueOf(preOffset)); + sliceVo.setBeginIdx(String.valueOf(preOffset.getCheckPoint())); sliceVo.setEndIdx(String.valueOf(offset)); - sliceVo.setPtnNum(1); - sliceVo.setPtn(0); sliceVo.setTotal(summarizedCheckPoint.size() - 1); - sliceVo.setEndpoint(endpoint); sliceVo.setNo(++index); sliceTaskList.add(sliceVo); - preOffset = offset; + preOffset = point; } return sliceTaskList; } - private List getCheckPoint(CheckPoint checkPoint, TableMetadata metadata) { - AutoSliceQueryStatement sliceStatement = factory.createSliceQueryStatement(checkPoint, metadata); - List checkPointList; + private List getCheckPoint(CheckPoint checkPoint, TableMetadata metadata) { + List checkPointList; + int sliceSize = ConfigCache.getIntValue(ConfigConstants.MAXIMUM_TABLE_SLICE_SIZE); + AutoSliceQueryStatement autoSliceStatement; + if (metadata.isUnionPrimary()) { + autoSliceStatement = factory.createUnionPrimarySliceQueryStatement(checkPoint); + } else { + autoSliceStatement = factory.createSliceQueryStatement(checkPoint, metadata); + } try { - checkPointList = sliceStatement.getCheckPoint(metadata, - ConfigCache.getIntValue(ConfigConstants.MAXIMUM_TABLE_SLICE_SIZE)); + checkPointList = autoSliceStatement.getCheckPoint(metadata, sliceSize); } catch (Exception ex) { LogUtils.error(log, "getCheckPoint error:", ex); return new ArrayList<>(); @@ -446,7 +534,8 @@ public class DataExtractServiceImpl implements DataExtractService { registerCheckPoint(task, endpoint); }); LogUtils.info(log, "tableRegisterCheckPoint finished"); - while (tableCheckPointCache.tableCount() != taskList.size()) { + long count = taskList.stream().filter(task -> task.getTableMetadata().isExistTableRows()).count(); + while (tableCheckPointCache.tableCount() != count) { ThreadUtil.sleepHalfSecond(); } checkPointManager.close(); @@ -457,16 +546,23 @@ public class DataExtractServiceImpl implements DataExtractService { private void registerCheckPoint(ExtractTask task, Endpoint endpoint) { try { String tableName = task.getTableName(); - LogUtils.info(log, "register check point [{}][{}]", endpoint, tableName); CheckPoint checkPoint = new CheckPoint(dataAccessService); - List checkPointList = getCheckPoint(checkPoint, task.getTableMetadata()); - if (checkPointList == null || checkPointList.size() <= 2) { - checkPointList = List.of(); - tableCheckPointCache.put(tableName, checkPointList); + TableMetadata tableMetadata = task.getTableMetadata(); + if (tableMetadata.isExistTableRows()) { + LogUtils.info(log, "register check point [{}][{}]", endpoint, tableName); + List checkPointList = getCheckPoint(checkPoint, tableMetadata); + if (checkPointList == null || checkPointList.size() <= 2) { + checkPointList = List.of(); + tableCheckPointCache.put(tableName, checkPointList); + } + CheckPointData checkPointData = new CheckPointData().setTableName(tableName) + .setColName(checkPoint.getSliceColumnName(tableMetadata)) + .setDigit(checkPoint.checkPkNumber(tableMetadata)) + .setCheckPointList(checkPointList); + checkPointManager.send(checkPointData); + } else { + tableCheckPointCache.put(tableName, List.of()); } - checkPointManager.send(new CheckPointData().setTableName(tableName) - .setDigit(checkPoint.checkPkNumber(task.getTableMetadata())) - .setCheckPointList(checkPointList)); } catch (Exception e) { log.error("register check point failed ", e); } @@ -476,18 +572,6 @@ public class DataExtractServiceImpl implements DataExtractService { return TASK_NAME_PREFIX.concat(tableName).concat("_slice_").concat(String.valueOf(index + 1)); } - private void registerTopic(ExtractTask task) { - int topicPartitions = TopicUtil.calcPartitions(task.getDivisionsTotalNumber()); - Endpoint currentEndpoint = extractProperties.getEndpoint(); - Topic topic; - if (Objects.equals(Endpoint.SOURCE, currentEndpoint)) { - topic = checkingFeignClient.sourceRegisterTopic(task.getTableName(), topicPartitions); - } else { - topic = checkingFeignClient.sinkRegisterTopic(task.getTableName(), topicPartitions); - } - task.setTopic(topic); - } - /** * Query table data * @@ -547,6 +631,7 @@ public class DataExtractServiceImpl implements DataExtractService { BeanUtils.copyProperties(extractProperties, config); database.setLowercaseTableNames(dataAccessService.queryLowerCaseTableNames()); config.setDatabase(database); + config.setMaxSliceSize(ConfigCache.getIntValue(ConfigConstants.MAXIMUM_TABLE_SLICE_SIZE)); return config; } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/TableRuleAdapterService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/TableRuleAdapterService.java index 768ebe6b3274a767e3d767785128f12d6867abc6..57bd4c41af53eb1d0db9c91a85a6532667ffe9bc 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/TableRuleAdapterService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/TableRuleAdapterService.java @@ -20,6 +20,7 @@ import org.opengauss.datachecker.common.constant.RuleConstants; import org.opengauss.datachecker.common.entry.common.Rule; import org.springframework.stereotype.Service; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -38,20 +39,23 @@ public class TableRuleAdapterService { private static final String WHITE = RuleConstants.RULE_WHITE; private static final String BLACK = RuleConstants.RULE_BLACK; private static final Map EXECUTORS = new HashMap<>(); + private static final Map REAL_NAME_EXECUTORS = new HashMap<>(); static { - EXECUTORS.put(WHITE, (patterns, table) -> patterns.stream() - .anyMatch(pattern -> pattern.matcher(table) - .matches())); - EXECUTORS.put(BLACK, (patterns, table) -> patterns.stream() - .noneMatch(pattern -> pattern.matcher(table) - .matches())); + EXECUTORS.put(WHITE, + (patterns, table) -> patterns.stream().anyMatch(pattern -> pattern.matcher(table).matches())); + EXECUTORS.put(BLACK, + (patterns, table) -> patterns.stream().noneMatch(pattern -> pattern.matcher(table).matches())); + REAL_NAME_EXECUTORS.put(WHITE, + (patterns, table) -> patterns.stream().anyMatch(pattern -> pattern.contains(table))); + REAL_NAME_EXECUTORS.put(BLACK, + (patterns, table) -> patterns.stream().noneMatch(pattern -> pattern.contains(table))); } /** * executeTableRule * - * @param rules rules + * @param rules rules * @param tableList tableList * @return filter list */ @@ -63,8 +67,8 @@ public class TableRuleAdapterService { final TableRuleExecutor tableRuleExecutor = EXECUTORS.get(ruleOne.getName()); final List patterns = buildRulePatterns(rules); return tableList.parallelStream() - .filter(table -> tableRuleExecutor.apply(patterns, table)) - .collect(Collectors.toList()); + .filter(table -> tableRuleExecutor.apply(patterns, table)) + .collect(Collectors.toList()); } /** @@ -79,19 +83,43 @@ public class TableRuleAdapterService { return true; } final Rule ruleOne = rules.get(0); + if (ruleOne.getText().contains(",")) { + List tableRules = buildRealNameRule(rules); + TableRealNameRuleExecutor realNameRuleExecutor = REAL_NAME_EXECUTORS.get(ruleOne.getName()); + return realNameRuleExecutor.apply(tableRules, table); + } final TableRuleExecutor tableRuleExecutor = EXECUTORS.get(ruleOne.getName()); final List patterns = buildRulePatterns(rules); return tableRuleExecutor.apply(patterns, table); } private List buildRulePatterns(List rules) { + return rules.stream().map(rule -> Pattern.compile(rule.getText())).collect(Collectors.toList()); + } + + private List buildRealNameRule(List rules) { return rules.stream() - .map(rule -> Pattern.compile(rule.getText())) - .collect(Collectors.toList()); + .map(Rule::getText) + .map(text -> text.split(",")) + .map(Arrays::asList) + .flatMap(List::stream) + .collect(Collectors.toList()); } @FunctionalInterface interface TableRuleExecutor { boolean apply(List tablePatterns, String table); } + + @FunctionalInterface + interface TableRealNameRuleExecutor { + /** + * Judge whether it matches according to the rule and the table name + * + * @param tablePatterns tablePatterns + * @param table table + * @return boolean + */ + boolean apply(List tablePatterns, String table); + } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/ExtractPointSwapManager.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/ExtractPointSwapManager.java index a35691fac73576d0b0f72eba51f447b81393d4ab..9ce8b450273daff55bd859ce433c5780d8abc77d 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/ExtractPointSwapManager.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/ExtractPointSwapManager.java @@ -17,13 +17,18 @@ package org.opengauss.datachecker.extract.slice; import com.alibaba.fastjson.JSONObject; +import cn.hutool.core.bean.BeanUtil; + import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.errors.TimeoutException; import org.apache.logging.log4j.Logger; import org.opengauss.datachecker.common.config.ConfigCache; import org.opengauss.datachecker.common.constant.Constants; +import org.opengauss.datachecker.common.entry.common.CheckPointBean; import org.opengauss.datachecker.common.entry.common.CheckPointData; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.enums.Endpoint; import org.opengauss.datachecker.common.util.IdGenerator; import org.opengauss.datachecker.common.util.LogUtils; @@ -33,9 +38,12 @@ import org.opengauss.datachecker.extract.config.KafkaConsumerConfig; import org.springframework.kafka.core.KafkaTemplate; import java.time.Duration; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -47,7 +55,9 @@ import java.util.stream.Collectors; */ public class ExtractPointSwapManager { private static final Logger log = LogUtils.getLogger(ExtractPointSwapManager.class); + private static final int MAX_RETRY_TIMES = 100; + private final Map> tablePointCache = new ConcurrentHashMap<>(); private final KafkaTemplate kafkaTemplate; private final KafkaConsumer kafkaConsumer; private String checkPointSwapTopicName = null; @@ -66,12 +76,50 @@ public class ExtractPointSwapManager { this.kafkaConsumer = kafkaConsumerConfig.createConsumer(IdGenerator.nextId36()); } + /** + * send check point to check endpoint by kafka + * + * @param checkPointData check point + */ public void send(CheckPointData checkPointData) { checkPointData.setEndpoint(endpoint); - kafkaTemplate.send(checkPointSwapTopicName, endpoint.getDescription(), JSONObject.toJSONString(checkPointData)); - LogUtils.info(log, "send check point [{}][{}]", checkPointSwapTopicName, checkPointData); + List checkPointList = checkPointData.getCheckPointList(); + int size = checkPointList.size(); + if (size == 0) { + CheckPointBean pointBean = new CheckPointBean(); + BeanUtil.copyProperties(checkPointData, pointBean); + pointBean.setCheckPoint(null); + pointBean.setSize(size); + kafkaTemplate.send(checkPointSwapTopicName, endpoint.getDescription(), JSONObject.toJSONString(pointBean)); + } else { + checkPointList.forEach(checkPoint -> { + CheckPointBean pointBean = new CheckPointBean(); + BeanUtil.copyProperties(checkPointData, pointBean); + pointBean.setSize(size); + pointBean.setCheckPoint(checkPoint); + sendMsg(checkPointSwapTopicName, endpoint.getDescription(), pointBean, 0); + }); + } + log.info("send checkPointData success, table :{} size:{}", checkPointData.getTableName(), size); + } + + private void sendMsg(String topic, String key, CheckPointBean tmpBean, int reTryTimes) { + try { + kafkaTemplate.send(topic, key, JSONObject.toJSONString(tmpBean)); + } catch (TimeoutException ex) { + if (reTryTimes > MAX_RETRY_TIMES) { + log.error("send msg to kafka timeout, topic: {} key: {} reTryTimes: {}", topic, key, reTryTimes); + } + ThreadUtil.sleepLongCircle(++reTryTimes); + sendMsg(topic, key, tmpBean, reTryTimes); + } } + /** + * poll check point from check endpoint by kafka; add check point in cache + * + * @param tableCheckPointCache cache + */ public void pollSwapPoint(TableCheckPointCache tableCheckPointCache) { executorService.submit(() -> { trySubscribe(); @@ -85,24 +133,24 @@ public class ExtractPointSwapManager { if (!records.isEmpty()) { records.forEach(record -> { if (Objects.equals(record.key(), Endpoint.CHECK.getDescription())) { - CheckPointData pointData = JSONObject.parseObject(record.value(), CheckPointData.class); - tableCheckPointCache.put(pointData.getTableName(), translateDigitPoint(pointData)); - deliveredCount.getAndIncrement(); - LogUtils.info(log, "swap summarized checkpoint of table [{}]:[{}] ", deliveredCount, - pointData.toString()); + CheckPointBean pointBean = JSONObject.parseObject(record.value(), CheckPointBean.class); + if (tablePointCache.containsKey(pointBean.getTableName())) { + tablePointCache.get(pointBean.getTableName()).add(pointBean); + } else { + List list = new LinkedList<>(); + list.add(pointBean); + tablePointCache.put(pointBean.getTableName(), list); + } } }); - ThreadUtil.sleepHalfSecond(); + retryTimesWait = 0; } else { LogUtils.info(log, "wait swap summarized checkpoint of table {}...", ++retryTimesWait); ThreadUtil.sleepCircle(retryTimesWait); } + processTablePoint(tableCheckPointCache, deliveredCount); } catch (Exception ex) { - if (Objects.equals("java.lang.InterruptedException", ex.getMessage())) { - LogUtils.warn(log, "kafka consumer stop by Interrupted"); - } else { - LogUtils.error(log, "pollSwapPoint ", ex); - } + LogUtils.info(log, "pollSwapPoint swap summarized checkpoint end {}", ex.getMessage()); } } LogUtils.warn(log, "close check point swap consumer {} :{}", checkPointSwapTopicName, @@ -111,11 +159,22 @@ public class ExtractPointSwapManager { }); } - private List translateDigitPoint(CheckPointData pointData) { - return pointData.isDigit() ? pointData.getCheckPointList() - .stream() - .map(obj -> Long.parseLong((String) obj)) - .collect(Collectors.toList()) : pointData.getCheckPointList(); + private void processTablePoint(TableCheckPointCache tableCheckPointCache, AtomicInteger deliveredCount) { + Iterator>> iterator = tablePointCache.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry> next = iterator.next(); + List value = next.getValue(); + CheckPointBean checkPointBean = value.get(0); + if (checkPointBean.getSize() == value.size()) { + List collect = value.stream() + .map(CheckPointBean::getCheckPoint) + .collect(Collectors.toList()); + tableCheckPointCache.add(next.getKey(), collect); + deliveredCount.getAndIncrement(); + iterator.remove(); + log.info("send checkpoint success, table: {}, size {}", next.getKey(), collect.size()); + } + } } private void trySubscribe() { @@ -144,9 +203,11 @@ public class ExtractPointSwapManager { this.checkPointSwapTopicName = String.format(Constants.SWAP_POINT_TOPIC_TEMP, process); } + /** + * close check point swap thread + */ public void close() { this.isCompletedSwapTablePoint = true; - this.executorService.shutdownNow(); } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/SliceStatusFeedbackService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/SliceStatusFeedbackService.java index f7194fe31b05137a4ae866818ce72c004a9ce71b..bd5eb32f729a0d296af7eac5efce904001da1462 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/SliceStatusFeedbackService.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/SliceStatusFeedbackService.java @@ -76,22 +76,26 @@ public class SliceStatusFeedbackService { */ public void feedback() { feedbackSender.submit(() -> { - Thread.currentThread() - .setName(FEEDBACK_THREAD_NAME); + Thread.currentThread().setName(FEEDBACK_THREAD_NAME); SliceExtend sliceExt = null; while (!isCompleted) { try { sliceExt = feedbackQueue.poll(); - if (Objects.isNull(sliceExt) && !isCompleted) { - ThreadUtil.sleepHalfSecond(); - continue; + if (Objects.isNull(sliceExt)) { + if (isCompleted) { + LogUtils.debug(log, "feedback slice status of is completed"); + } else { + ThreadUtil.sleepOneSecond(); + } + } else { + LogUtils.debug(log, "feedback slice status of table [{}]", sliceExt); + checkingClient.refreshRegisterSlice(sliceExt); } - checkingClient.refreshRegisterSlice(sliceExt); - LogUtils.debug(log,"feedback slice status of table [{}]", sliceExt); } catch (Exception ex) { - LogUtils.error(log,"feedback slice status error {}", sliceExt, ex); + LogUtils.error(log, "feedback slice status error {}", sliceExt, ex); } } + LogUtils.debug(log, "feedback slice status of is completed and exited"); }); } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcSliceProcessor.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcSliceProcessor.java index 450f8683d80b49a842d8e5653f05c4f44bc9f390..4efc0b92f9a2aaa42ca131c73f6f55741f3c2b35 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcSliceProcessor.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcSliceProcessor.java @@ -17,6 +17,8 @@ package org.opengauss.datachecker.extract.slice.process; import com.alibaba.druid.pool.DruidDataSource; +import lombok.Getter; + import org.apache.logging.log4j.Logger; import org.opengauss.datachecker.common.config.ConfigCache; import org.opengauss.datachecker.common.constant.ConfigConstants; @@ -42,8 +44,8 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; -import java.sql.SQLException; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.BlockingQueue; @@ -60,6 +62,8 @@ import java.util.concurrent.atomic.AtomicInteger; */ public class JdbcSliceProcessor extends AbstractSliceProcessor { private static final Logger log = LogUtils.getLogger(JdbcSliceProcessor.class); + private static final int MAX_RETRY_TIMES = 60; + private final JdbcDataOperations jdbcOperation; private final AtomicInteger rowCount = new AtomicInteger(0); private final DruidDataSource dataSource; @@ -96,7 +100,7 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { sliceExtend.setStatus(-1); LogUtils.error(log, "table slice [{}] is error", slice.toSimpleString(), ex); } finally { - LogUtils.info(log, "table slice [{}] is finally ", slice.toSimpleString()); + LogUtils.info(log, "table slice [{} count {}] is finally ", slice.toSimpleString(), rowCount.get()); feedbackStatus(sliceExtend); context.saveProcessing(slice); } @@ -119,8 +123,10 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { private void executeSliceQueryStatementPage(TableMetadata tableMetadata, SliceExtend sliceExtend) { // 分片数据统计 UnionPrimarySliceQueryStatement sliceStatement = context.createSlicePageQueryStatement(); - QuerySqlEntry sliceCountSql = sliceStatement.buildSliceCount(tableMetadata, slice); - int sliceCount = querySliceRowTotalCount(sliceExtend, sliceCountSql); + int sliceCount = (int) slice.getRowCountOfInIds(); + if (slice.getRowCountOfInIds() == 0) { + return; + } QuerySqlEntry baseSliceSql = sliceStatement.buildSlice(tableMetadata, slice); List pageStatementList = sliceStatement.buildPageStatement(baseSliceSql, sliceCount, slice.getFetchSize()); @@ -128,9 +134,9 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { Connection connection = null; try { // 申请数据库链接 - long estimatedRowCount = slice.isSlice() ? slice.getFetchSize() : tableMetadata.getTableRows(); + long estimatedRowCount = Math.min(sliceCount, slice.getFetchSize()); long estimatedMemorySize = estimatedMemorySize(tableMetadata.getAvgRowLength(), estimatedRowCount); - connection = jdbcOperation.tryConnectionAndClosedAutoCommit(estimatedMemorySize, dataSource); + connection = jdbcOperation.tryConnectionAndClosedAutoCommit(estimatedMemorySize); // 获取连接，准备查询分片数据： 并开启数据异步处理线程 sliceSender = createSliceResultSetSender(tableMetadata); sliceSender.setRecordSendKey(slice.getName()); @@ -142,50 +148,95 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { // 开始查询数据，并将结果推送到异步处理线程中。 boolean isFirstStatement = true; long startOffset = 0L; + int idx = 0; for (String pageStatement : pageStatementList) { + log.debug("executeSliceQueryStatementPage : {} : {}", ++idx, maskQuery(pageStatement)); + QueryParameters queryParameters = new QueryParameters(0, 0); if (isFirstStatement) { // only use first page statement's start offset - startOffset = pageQueryUnionPrimarySlice(pageStatement, connection, sliceSender, asyncHandler); + startOffset = statementQuery(pageStatement, connection, sliceSender, asyncHandler, queryParameters); } else { // other page statement's start offset is ignored - pageQueryUnionPrimarySlice(pageStatement, connection, sliceSender, asyncHandler); + statementQuery(pageStatement, connection, sliceSender, asyncHandler, queryParameters); } isFirstStatement = false; } + log.info("executeSliceQueryStatementPage : {} execute statement end", slice.getName()); sliceExtend.setStartOffset(startOffset); waitToStopAsyncHandlerAndResources(asyncHandler); updateExtendSliceOffsetAndCount(sliceExtend, rowCount.get(), offsetList); + log.info("executeSliceQueryStatementPage : {} async send end", slice.getName()); } catch (Exception ex) { LogUtils.error(log, "slice [{}] has exception :", slice.getName(), ex); throw new ExtractDataAccessException(ex.getMessage()); } finally { - ConnectionMgr.close(connection, null, null); if (sliceSender != null) { sliceSender.agentsClosed(); } jdbcOperation.releaseConnection(connection); - LogUtils.info(log, "query union primary slice and send data {} Count:{}", sliceExtend.getName(), - rowCount.get()); } } - private long pageQueryUnionPrimarySlice(String pageStatement, Connection connection, - SliceResultSetSender sliceSender, AsyncDataHandler asyncHandler) throws SQLException, InterruptedException { - long startOffset; - PreparedStatement ps = connection.prepareStatement(pageStatement); - ps.setFetchSize(FETCH_SIZE); - ResultSet resultSet = ps.executeQuery(); - startOffset = sliceSender.checkOffsetEnd(); - ResultSetMetaData rsmd = resultSet.getMetaData(); - while (resultSet.next()) { - this.rowCount.incrementAndGet(); - if (asyncHandler.isSenderBusy()) { - Thread.sleep(100); + /** + * only used by log print + * + * @param pageStatement sql statement + * @return mask statement + */ + private String maskQuery(String pageStatement) { + String[] split = pageStatement.toLowerCase(Locale.ROOT).split(" from "); + return "select * from " + split[1]; + } + + private long statementQuery(String pageStatement, Connection connection, SliceResultSetSender sliceSender, + AsyncDataHandler asyncHandler, QueryParameters queryParameters) { + long startOffset = -1L; + ExecutionStage executionStage = ExecutionStage.PREPARE; + PreparedStatement ps = null; + ResultSet resultSet = null; + int rsIdx = 0; + try { + executionStage = ExecutionStage.EXECUTE; + ps = connection.prepareStatement(pageStatement); + ps.setFetchSize(FETCH_SIZE); + resultSet = ps.executeQuery(); + startOffset = sliceSender.checkOffsetEnd(); + ResultSetMetaData rsmd = resultSet.getMetaData(); + executionStage = ExecutionStage.FETCH; + while (resultSet.next()) { + if (rsIdx >= queryParameters.getResultSetIdx()) { + this.rowCount.incrementAndGet(); + if (asyncHandler.isSenderBusy()) { + Thread.sleep(100); + } + asyncHandler.addRow(sliceSender.resultSet(rsmd, resultSet)); + } + rsIdx++; + } + executionStage = ExecutionStage.CLOSE; + // 数据发送到异步处理线程中，关闭ps与rs + ConnectionMgr.close(null, ps, resultSet); + } catch (Exception ex) { + log.error("execute query {} executionStage: {} error,retry cause : {}", slice.toSimpleString(), + executionStage, ex.getMessage()); + if (Objects.equals(executionStage, ExecutionStage.CLOSE)) { + ConnectionMgr.close(null, ps, resultSet); + } else { + ConnectionMgr.close(connection, ps, resultSet); + if (queryParameters.getRetryTimes() <= MAX_RETRY_TIMES) { + connection = jdbcOperation.tryConnectionAndClosedAutoCommit(0); + ++queryParameters.retryTimes; + queryParameters.resultSetIdx = rsIdx; + startOffset = statementQuery(pageStatement, connection, sliceSender, asyncHandler, queryParameters); + } else { + log.error("execute query {} retry {} times error ,cause by ", maskQuery(pageStatement), + queryParameters.getRetryTimes(), ex); + throw new ExtractDataAccessException( + "execute query " + maskQuery(pageStatement) + " retry " + MAX_RETRY_TIMES + + " times error ,cause by " + ex.getMessage()); + } } - asyncHandler.addRow(sliceSender.resultSet(rsmd, resultSet)); } - // 数据发送到异步处理线程中，关闭ps与rs - ConnectionMgr.close(null, ps, resultSet); return startOffset; } @@ -197,33 +248,12 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { throw new ExtractDataAccessException("slice data async handler is interrupted"); } } - - private int querySliceRowTotalCount(SliceExtend sliceExtend, QuerySqlEntry sliceCountSql) { - int sliceCount = 0; - try (Connection connection = jdbcOperation.tryConnectionAndClosedAutoCommit(1L, dataSource); - PreparedStatement ps = connection.prepareStatement(sliceCountSql.getSql()); - ResultSet resultSet = ps.executeQuery();) { - if (resultSet.next()) { - sliceCount = resultSet.getInt(1); - } - } catch (SQLException ex) { - log.error("execute slice count query error ", ex); - throw new ExtractDataAccessException("execute slice count query error"); - } - log.info("query union primary table slice {} Count:{}", sliceExtend.getName(), sliceCount); - return sliceCount; - } - private void executeQueryStatement(QuerySqlEntry sqlEntry, TableMetadata tableMetadata, SliceExtend sliceExtend) { SliceResultSetSender sliceSender = null; Connection connection = null; PreparedStatement ps = null; ResultSet resultSet = null; try { - // 申请数据库链接 - long estimatedRowCount = slice.isSlice() ? slice.getFetchSize() : tableMetadata.getTableRows(); - long estimatedMemorySize = estimatedMemorySize(tableMetadata.getAvgRowLength(), estimatedRowCount); - connection = jdbcOperation.tryConnectionAndClosedAutoCommit(estimatedMemorySize, dataSource); // 获取连接，准备查询分片数据： 并开启数据异步处理线程 List offsetList = new CopyOnWriteArrayList<>(); List>> batchFutures = new CopyOnWriteArrayList<>(); @@ -232,27 +262,16 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { AsyncDataHandler asyncHandler = new AsyncDataHandler(batchFutures, sliceSender, offsetList); asyncHandler.start(); context.asyncSendSlice(asyncHandler); + // 申请数据库链接 + long estimatedRowCount = slice.isSlice() ? slice.getFetchSize() : tableMetadata.getTableRows(); + long estimatedMemorySize = estimatedMemorySize(tableMetadata.getAvgRowLength(), estimatedRowCount); + connection = jdbcOperation.tryConnectionAndClosedAutoCommit(estimatedMemorySize); // 开始查询数据，并将结果推送到异步处理线程中。 - ps = connection.prepareStatement(sqlEntry.getSql()); - ps.setFetchSize(FETCH_SIZE); - resultSet = ps.executeQuery(); - sliceExtend.setStartOffset(sliceSender.checkOffsetEnd()); - ResultSetMetaData rsmd = resultSet.getMetaData(); - while (resultSet.next()) { - this.rowCount.incrementAndGet(); - if (asyncHandler.isSenderBusy()) { - Thread.sleep(100); - } - // 数据发送到异步处理线程 - asyncHandler.addRow(sliceSender.resultSet(rsmd, resultSet)); - } - // 全部分片查询处理完成，关闭数据库连接，并关闭异步数据处理线程 ，关闭ps与rs - try { - ConnectionMgr.close(null, ps, resultSet); - asyncHandler.waitToStop(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + QueryParameters parameters = new QueryParameters(0, 0); + long startOffset = statementQuery(sqlEntry.getSql(), connection, sliceSender, asyncHandler, parameters); + sliceExtend.setStartOffset(startOffset); + // 等待分片查询处理完成，关闭数据库连接，并关闭异步数据处理线程 ，关闭ps与rs + waitToStopAsyncHandlerAndResources(asyncHandler); updateExtendSliceOffsetAndCount(sliceExtend, rowCount.get(), offsetList); } catch (Exception ex) { LogUtils.error(log, "slice [{}] has exception :", slice.getName(), ex); @@ -267,6 +286,26 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { } } + /** + * statement query parameters + */ + @Getter + class QueryParameters { + private int retryTimes; + private int resultSetIdx; + + /** + * build statement query paramters + * + * @param retryTimes retry times + * @param resultSetIdx result set idx + */ + public QueryParameters(int retryTimes, int resultSetIdx) { + this.retryTimes = retryTimes; + this.resultSetIdx = resultSetIdx; + } + } + /** * async data handler thread */ @@ -359,3 +398,25 @@ public class JdbcSliceProcessor extends AbstractSliceProcessor { sliceExtend.setCount(rowCount); } } + +/** + * execution stage + */ +enum ExecutionStage { + /** + * prepare + */ + PREPARE, + /** + * execute + */ + EXECUTE, + /** + * fetch + */ + FETCH, + /** + * close + */ + CLOSE +} \ No newline at end of file diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcTableProcessor.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcTableProcessor.java index 64cfa8b35df73812c7b2c762e6b48b701f0ae076..7216a6c6442a2b52771f10f784ff8d70c2f409de 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcTableProcessor.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/slice/process/JdbcTableProcessor.java @@ -150,7 +150,7 @@ public class JdbcTableProcessor extends AbstractTableProcessor { int fetchSize = getFetchSize(); try { long estimatedSize = estimatedMemorySize(tableMetadata.getAvgRowLength(), fetchSize); - connection = jdbcOperation.tryConnectionAndClosedAutoCommit(estimatedSize, dataSource); + connection = jdbcOperation.tryConnectionAndClosedAutoCommit(estimatedSize); QuerySqlEntry sqlEntry = getFullQuerySqlEntry(); log.info(" {} , {}", table, sqlEntry.toString()); List offsetList = new LinkedList<>(); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/CheckPoint.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/CheckPoint.java index 674e50d80a8675b4dc5da029cbe2050e020de109..eb7f3800fec3c1af58ac98c85215edef1f953af0 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/CheckPoint.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/CheckPoint.java @@ -16,16 +16,16 @@ package org.opengauss.datachecker.extract.task; import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.pool.DruidPooledConnection; +import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.Logger; import org.opengauss.datachecker.common.config.ConfigCache; import org.opengauss.datachecker.common.constant.ConfigConstants; import org.opengauss.datachecker.common.entry.common.DataAccessParam; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.enums.DataBaseType; import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; import org.opengauss.datachecker.common.entry.extract.TableMetadata; -import org.opengauss.datachecker.common.exception.ExtractDataAccessException; import org.opengauss.datachecker.common.util.LogUtils; import org.opengauss.datachecker.common.util.SqlUtil; import org.opengauss.datachecker.extract.data.access.DataAccessService; @@ -34,7 +34,7 @@ import org.opengauss.datachecker.extract.util.MetaDataUtil; import org.springframework.util.StopWatch; import java.sql.Connection; -import java.sql.SQLException; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -70,7 +70,7 @@ public class CheckPoint { * @param slice slice * @return check point */ - public List initCheckPointList(TableMetadata tableMetadata, int slice) { + public List initCheckPointList(TableMetadata tableMetadata, int slice) { if (slice <= 0) { return new LinkedList<>(); } @@ -92,16 +92,11 @@ public class CheckPoint { stopWatch.stop(); LogUtils.info(log, "init check-point-list table [{}]:[{}] ", tableName, stopWatch.shortSummary()); ConnectionMgr.close(connection); - return checkPointList; + return checkPointList.stream().map(o -> new PointPair(o, 0)).collect(Collectors.toList()); } - private DruidPooledConnection getConnection() { - try { - return dataSource.getConnection(); - } catch (SQLException throwables) { - throwables.printStackTrace(); - throw new ExtractDataAccessException("get connection error"); - } + private Connection getConnection() { + return ConnectionMgr.getConnection(); } private void addCheckList(List checkList, Object value) { @@ -111,12 +106,11 @@ public class CheckPoint { } public boolean checkPkNumber(TableMetadata tableMetadata) { - ColumnsMetaData pkColumn = tableMetadata.getPrimaryMetas().get(0); - return MetaDataUtil.isDigitPrimaryKey(pkColumn); + return MetaDataUtil.isDigitPrimaryKey(tableMetadata.getSliceColumn()); } private String getPkName(TableMetadata tableMetadata) { - return tableMetadata.getPrimaryMetas().get(0).getColumnName(); + return tableMetadata.getSliceColumn().getColumnName(); } public Long[][] translateBetween(List checkPointList) { @@ -153,4 +147,63 @@ public class CheckPoint { ConnectionMgr.close(connection, null, null); return Long.parseLong(maxId); } + + /** + * 初始化联合主键表检查点 + * + * @param tableMetadata tableMetadata + * @return 检查点列表 + */ + public List initUnionPrimaryCheckPointList(TableMetadata tableMetadata) { + List checkPointList = new ArrayList<>(); + List primaryList = tableMetadata.getPrimaryMetas(); + for (ColumnsMetaData unionKey : primaryList) { + List tmp = queryUnionKeyList(unionKey); + if (CollectionUtils.isEmpty(tmp)) { + tableMetadata.setSliceColumn(unionKey); + break; + } + if (CollectionUtils.isEmpty(checkPointList)) { + checkPointList = tmp; + tableMetadata.setSliceColumn(unionKey); + } else { + if (checkPointList.size() > tmp.size()) { + checkPointList = tmp; + tableMetadata.setSliceColumn(unionKey); + } + } + } + return checkPointList; + } + + private List queryUnionKeyList(ColumnsMetaData unionKey) { + String colName = unionKey.getColumnName(); + String schema = unionKey.getSchema(); + String tableName = unionKey.getTableName(); + DataBaseType dataBaseType = ConfigCache.getValue(ConfigConstants.DATA_BASE_TYPE, DataBaseType.class); + DataAccessParam param = new DataAccessParam().setSchema(SqlUtil.escape(schema, dataBaseType)) + .setName(SqlUtil.escape(tableName, dataBaseType)) + .setColName(SqlUtil.escape(colName, dataBaseType)); + try (Connection connection = getConnection()) { + return dataAccessService.queryUnionFirstPrimaryCheckPointList(connection, param); + } catch (Exception e) { + log.error("query union primary check point list error {}", e.getMessage()); + } + return new LinkedList<>(); + } + + /** + * 获取分片列名 + * + * @param tableMetadata 表元数据 + * @return 分片列名 + */ + public String getSliceColumnName(TableMetadata tableMetadata) { + ColumnsMetaData sliceColumn = tableMetadata.getSliceColumn(); + if (Objects.nonNull(sliceColumn)) { + return sliceColumn.getColumnName(); + } else { + return ""; + } + } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/AutoSliceQueryStatement.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/AutoSliceQueryStatement.java index b9040edb18f84650c783f677db48057b365f8290..dd54478be62b60f0a11e01ee0c7f6d0bab8db7e3 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/AutoSliceQueryStatement.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/AutoSliceQueryStatement.java @@ -15,6 +15,7 @@ package org.opengauss.datachecker.extract.task.sql; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import java.util.List; @@ -52,5 +53,5 @@ public interface AutoSliceQueryStatement extends QueryStatement { * @param slice slice * @return list checkPoint list */ - List getCheckPoint(TableMetadata tableMetadata, int slice); + List getCheckPoint(TableMetadata tableMetadata, int slice); } \ No newline at end of file diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/QueryStatementFactory.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/QueryStatementFactory.java index c960b454b04b8bf136745b8a5e924cc8ae8033db..2f60f73b5ab10a1e4367c63e7211f4bbc41dc181 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/QueryStatementFactory.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/QueryStatementFactory.java @@ -57,6 +57,16 @@ public class QueryStatementFactory { return new UnionPrimarySliceQueryStatement(); } + /** + * create UnionPrimaryAutoSliceQueryStatement + * + * @param checkPoint checkPoint + * @return A new AutoSliceQueryStatement instance. + */ + public AutoSliceQueryStatement createUnionPrimarySliceQueryStatement(CheckPoint checkPoint) { + return new UnionPrimaryAutoSliceQueryStatement(checkPoint); + } + /** * create FullQueryStatement * diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SelectSqlBuilder.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SelectSqlBuilder.java index f3e16b849b59f78e1438edaf730b2f9cb9716500..c208b1fd45789875ab63a7c5e0030b09ee11fcf4 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SelectSqlBuilder.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SelectSqlBuilder.java @@ -98,9 +98,10 @@ public class SelectSqlBuilder { private boolean isFullCondition; private boolean isHalfOpenHalfClosed = true; private boolean isFirst = false; + private boolean isDigit = false; private boolean isEnd = false; private boolean isCsvMode = false; - private String countSnippet = "count(1)"; + private List inIds = null; /** * Table fragment query SQL Statement Builder @@ -154,6 +155,17 @@ public class SelectSqlBuilder { return this; } + /** + * set current key is digit + * + * @param isDigit is digit + * @return builder + */ + public SelectSqlBuilder isDigit(boolean isDigit) { + this.isDigit = isDigit; + return this; + } + public SelectSqlBuilder isFullCondition(boolean isFullCondition) { this.isFullCondition = isFullCondition; return this; @@ -186,41 +198,42 @@ public class SelectSqlBuilder { Assert.notEmpty(columnsMetas, Message.COLUMN_METADATA_EMPTY_NOT_TO_BUILD_SQL); final ConditionLimit conditionLimit = tableMetadata.getConditionLimit(); if (Objects.nonNull(conditionLimit)) { - return buildSelectSqlConditionLimit(tableMetadata, conditionLimit, null); + return buildSelectSqlConditionLimit(tableMetadata, conditionLimit); } else if (isDivisions) { - return buildSelectSqlWherePrimary(tableMetadata, null); + if (tableMetadata.isUnionPrimary()) { + return buildSelectSqlWhereInPrimary(tableMetadata); + } else { + return buildSelectSqlWherePrimary(tableMetadata); + } } else { - return buildSelectSqlOffsetZero(columnsMetas, tableMetadata.getTableName(), null); + return buildSelectSqlOffsetZero(columnsMetas, tableMetadata.getTableName()); } } - /** - * select row count sql builder - * - * @return sql - */ - public String countBuilder() { - Assert.isTrue(Objects.nonNull(tableMetadata), Message.TABLE_METADATA_NULL_NOT_TO_BUILD_SQL); + private String buildSelectSqlWhereInPrimary(TableMetadata tableMetadata) { List columnsMetas = tableMetadata.getColumnsMetas(); - Assert.notEmpty(columnsMetas, Message.COLUMN_METADATA_EMPTY_NOT_TO_BUILD_SQL); - final ConditionLimit conditionLimit = tableMetadata.getConditionLimit(); - if (Objects.nonNull(conditionLimit)) { - return buildSelectSqlConditionLimit(tableMetadata, conditionLimit, countSnippet); - } else if (isDivisions) { - return buildSelectSqlWherePrimary(tableMetadata, countSnippet); - } else { - return buildSelectSqlOffsetZero(columnsMetas, tableMetadata.getTableName(), countSnippet); - } + String schemaEscape = escape(schema, dataBaseType); + String tableName = escape(tableMetadata.getTableName(), dataBaseType); + String columnNames = getColumnNameList(columnsMetas, dataBaseType); + String primaryKey = escape(tableMetadata.getPrimaryMetas().get(0).getColumnName(), dataBaseType); + final String orderBy = getOrderBy(tableMetadata.getPrimaryMetas(), dataBaseType); + String pkCondition = primaryKey + "in (" + inIds.stream() + .map(id -> isDigit ? id : "'" + id + "'") + .collect(Collectors.joining(",")) + ")"; + return QUERY_WHERE_BETWEEN.replace(COLUMN, columnNames) + .replace(SCHEMA, schemaEscape) + .replace(TABLE_NAME, tableName) + .replace(PK_CONDITION, pkCondition) + .replace(ORDER_BY, orderBy); } - private String buildSelectSqlWherePrimary(TableMetadata tableMetadata, String countSnippet) { + private String buildSelectSqlWherePrimary(TableMetadata tableMetadata) { List columnsMetas = tableMetadata.getColumnsMetas(); String schemaEscape = escape(schema, dataBaseType); String tableName = escape(tableMetadata.getTableName(), dataBaseType); - boolean isCountSnippet = StringUtils.isNotEmpty(countSnippet); - String columnNames = isCountSnippet ? countSnippet : getColumnNameList(columnsMetas, dataBaseType); + String columnNames = getColumnNameList(columnsMetas, dataBaseType); String primaryKey = escape(tableMetadata.getPrimaryMetas().get(0).getColumnName(), dataBaseType); - final String orderBy = isCountSnippet ? "" : getOrderBy(tableMetadata.getPrimaryMetas(), dataBaseType); + final String orderBy = getOrderBy(tableMetadata.getPrimaryMetas(), dataBaseType); String pkCondition; if (StringUtils.isNotEmpty(seqStart) && StringUtils.isNotEmpty(seqEnd)) { pkCondition = getPkCondition(primaryKey); @@ -235,6 +248,9 @@ public class SelectSqlBuilder { } private String getNumberPkCondition(String primaryKey) { + if (start == offset) { + return primaryKey + " = " + start; + } if (isFullCondition) { return getNumberPkConditionFull(primaryKey); } @@ -264,6 +280,9 @@ public class SelectSqlBuilder { } private String getPkCondition(String primaryKey) { + if (StringUtils.equalsIgnoreCase(seqStart, seqEnd)) { + return primaryKey + " = '" + seqStart + "'"; + } if (isFullCondition) { return getPkConditionFull(primaryKey); } @@ -284,14 +303,12 @@ public class SelectSqlBuilder { } } - private String buildSelectSqlConditionLimit(TableMetadata tableMetadata, ConditionLimit conditionLimit, - String countSnippet) { + private String buildSelectSqlConditionLimit(TableMetadata tableMetadata, ConditionLimit conditionLimit) { List columnsMetas = tableMetadata.getColumnsMetas(); - boolean isCountSnippet = StringUtils.isNotEmpty(countSnippet); - String columnNames = isCountSnippet ? countSnippet : getColumnNameList(columnsMetas, dataBaseType); + String columnNames = getColumnNameList(columnsMetas, dataBaseType); final String schemaEscape = escape(schema, dataBaseType); final String tableName = escape(tableMetadata.getTableName(), dataBaseType); - final String orderBy = isCountSnippet ? "" : getOrderBy(tableMetadata.getPrimaryMetas(), dataBaseType); + final String orderBy = getOrderBy(tableMetadata.getPrimaryMetas(), dataBaseType); SqlGenerateMeta sqlGenerateMeta = new SqlGenerateMeta(schemaEscape, tableName, columnNames, orderBy, conditionLimit.getStart(), conditionLimit.getOffset()); return getSqlGenerate(dataBaseType).replace(sqlGenerateMeta); @@ -329,9 +346,8 @@ public class SelectSqlBuilder { return SqlUtil.escape(content, dataBaseType); } - private String buildSelectSqlOffsetZero(List columnsMetas, String tableName, String countSnippet) { - boolean isCountSnippet = StringUtils.isNotEmpty(countSnippet); - String columnNames = isCountSnippet ? countSnippet : getColumnNameList(columnsMetas, dataBaseType); + private String buildSelectSqlOffsetZero(List columnsMetas, String tableName) { + String columnNames = getColumnNameList(columnsMetas, dataBaseType); String schemaEscape = escape(schema, dataBaseType); SqlGenerateMeta sqlGenerateMeta = new SqlGenerateMeta(schemaEscape, escape(tableName, dataBaseType), columnNames); @@ -357,6 +373,17 @@ public class SelectSqlBuilder { this.isEnd = end; } + /** + * union key table set slice key for sql select where in + * + * @param inIds ids + * @param isDigit is Digit + */ + public void inIds(List inIds, boolean isDigit) { + this.inIds = inIds; + this.isDigit = isDigit; + } + @Getter static class SqlGenerateMeta { private String schema; diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimaryAutoIncrementAutoSliceQueryStatement.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimaryAutoIncrementAutoSliceQueryStatement.java index bccc4066f45112ddeec844b45251397d08372136..5b01d366186e1d08b931b59ffb42c569f8daeaef 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimaryAutoIncrementAutoSliceQueryStatement.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimaryAutoIncrementAutoSliceQueryStatement.java @@ -15,6 +15,7 @@ package org.opengauss.datachecker.extract.task.sql; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.opengauss.datachecker.common.util.TaskUtilHelper; import org.opengauss.datachecker.extract.task.CheckPoint; @@ -62,7 +63,7 @@ public class SinglePrimaryAutoIncrementAutoSliceQueryStatement implements AutoSl } @Override - public List getCheckPoint(TableMetadata tableMetadata, int slice) { + public List getCheckPoint(TableMetadata tableMetadata, int slice) { return null; } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimaryAutoSliceQueryStatement.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimaryAutoSliceQueryStatement.java index 4d266633665fbd8aa4f5b86d16107f12273e8d44..e206a6554566845063c8056cbc41f76e281eba6e 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimaryAutoSliceQueryStatement.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimaryAutoSliceQueryStatement.java @@ -15,13 +15,11 @@ package org.opengauss.datachecker.extract.task.sql; -import org.opengauss.datachecker.common.config.ConfigCache; +import org.opengauss.datachecker.common.entry.common.PointPair; import org.opengauss.datachecker.common.entry.extract.TableMetadata; import org.opengauss.datachecker.extract.task.CheckPoint; -import java.util.LinkedList; import java.util.List; -import java.util.stream.IntStream; /** * single primary slice query statement @@ -44,41 +42,16 @@ public class SinglePrimaryAutoSliceQueryStatement implements AutoSliceQueryState @Override public List builderByTaskOffset(TableMetadata tableMetadata, int slice) { - List querySqlList = new LinkedList<>(); - List checkPointList = singlePrimaryCheckPoint.initCheckPointList(tableMetadata, slice); - final Object[][] taskOffset; - if (singlePrimaryCheckPoint.checkPkNumber(tableMetadata)) { - taskOffset = singlePrimaryCheckPoint.translateBetween(checkPointList); - } else { - taskOffset = singlePrimaryCheckPoint.translateBetweenString(checkPointList); - } - final SelectSqlBuilder sqlBuilder = new SelectSqlBuilder(tableMetadata); - sqlBuilder.isDivisions(taskOffset.length > 1) - .isCsvMode(ConfigCache.isCsvMode()); - String tableName = tableMetadata.getTableName(); - IntStream.range(0, taskOffset.length).forEach(idx -> { - sqlBuilder.isFullCondition(idx == taskOffset.length - 1).offset(taskOffset[idx][0], taskOffset[idx][1]); - QuerySqlEntry entry = - new QuerySqlEntry(tableName, sqlBuilder.builder(), taskOffset[idx][0], taskOffset[idx][1]); - querySqlList.add(entry); - }); - return querySqlList; + return null; } @Override public Object[][] builderSlice(TableMetadata tableMetadata, int slice) { - List checkPointList = singlePrimaryCheckPoint.initCheckPointList(tableMetadata, slice); - final Object[][] taskOffset; - if (singlePrimaryCheckPoint.checkPkNumber(tableMetadata)) { - taskOffset = singlePrimaryCheckPoint.translateBetween(checkPointList); - } else { - taskOffset = singlePrimaryCheckPoint.translateBetweenString(checkPointList); - } - return taskOffset; + return null; } @Override - public List getCheckPoint(TableMetadata tableMetadata, int slice) { + public List getCheckPoint(TableMetadata tableMetadata, int slice) { return singlePrimaryCheckPoint.initCheckPointList(tableMetadata, slice); } } diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimarySliceQueryStatement.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimarySliceQueryStatement.java index 607e0e23a5fcf11a3cdd288ecf15cf6fd5c9d2f4..68980960a855c6dbab0fc93c698593e2a5ebf12e 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimarySliceQueryStatement.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/SinglePrimarySliceQueryStatement.java @@ -52,6 +52,7 @@ public class SinglePrimarySliceQueryStatement implements SliceQueryStatement { sqlBuilder.isCsvMode(ConfigCache.isCsvMode()); ColumnsMetaData primaryKey = tableMetadata.getSinglePrimary(); boolean isDigit = MetaDataUtil.isDigitPrimaryKey(primaryKey); + sqlBuilder.isDigit(isDigit); Object offset = translateOffset(isDigit, slice.getBeginIdx()); Object endOffset = translateOffset(isDigit, slice.getEndIdx()); sqlBuilder.offset(offset, endOffset); diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/UnionPrimaryAutoSliceQueryStatement.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/UnionPrimaryAutoSliceQueryStatement.java new file mode 100644 index 0000000000000000000000000000000000000000..b1d0a5c91772a408870dc36380de7e92f7a96f62 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/UnionPrimaryAutoSliceQueryStatement.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +package org.opengauss.datachecker.extract.task.sql; + +import org.opengauss.datachecker.common.entry.common.PointPair; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.opengauss.datachecker.extract.task.CheckPoint; + +import java.util.List; + +/** + * Union primary slice query statement + * + * @author ：wangchao + * @date ：Created in 2023/8/9 + * @since ：11 + */ +public class UnionPrimaryAutoSliceQueryStatement implements AutoSliceQueryStatement { + private final CheckPoint primaryCheckPoint; + + /** + * create SinglePrimarySliceQueryStatement + * + * @param checkPoint checkPoint + */ + public UnionPrimaryAutoSliceQueryStatement(CheckPoint checkPoint) { + this.primaryCheckPoint = checkPoint; + } + + @Override + public List builderByTaskOffset(TableMetadata tableMetadata, int slice) { + return null; + } + + @Override + public Object[][] builderSlice(TableMetadata tableMetadata, int slice) { + return null; + } + + @Override + public List getCheckPoint(TableMetadata tableMetadata, int slice) { + return primaryCheckPoint.initUnionPrimaryCheckPointList(tableMetadata); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/UnionPrimarySliceQueryStatement.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/UnionPrimarySliceQueryStatement.java index ca917e850a08d6f4b37548029057f7338b910812..90deeb1d8f7882bee914dadde0a9a5cc0c8cf80f 100644 --- a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/UnionPrimarySliceQueryStatement.java +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/sql/UnionPrimarySliceQueryStatement.java @@ -44,28 +44,6 @@ public class UnionPrimarySliceQueryStatement implements SliceQueryStatement { this.isHalfOpenHalfClosed = !Objects.equals(ConfigCache.getCheckMode(), CheckMode.CSV); } - /** - * build slice count sql entry - * - * @param tableMetadata tableMetadata - * @param slice slice - * @return sql entry - */ - public QuerySqlEntry buildSliceCount(TableMetadata tableMetadata, SliceVo slice) { - final SelectSqlBuilder sqlBuilder = new SelectSqlBuilder(tableMetadata); - sqlBuilder.isDivisions(slice.getTotal() > 1); - sqlBuilder.isFirstCondition(slice.getNo() == 1); - sqlBuilder.isEndCondition(slice.getNo() == slice.getTotal()); - sqlBuilder.isHalfOpenHalfClosed(isHalfOpenHalfClosed); - sqlBuilder.isCsvMode(ConfigCache.isCsvMode()); - ColumnsMetaData primaryKey = tableMetadata.getSinglePrimary(); - boolean isDigit = MetaDataUtil.isDigitPrimaryKey(primaryKey); - Object offset = translateOffset(isDigit, slice.getBeginIdx()); - Object endOffset = translateOffset(isDigit, slice.getEndIdx()); - sqlBuilder.offset(offset, endOffset); - return new QuerySqlEntry(slice.getTable(), sqlBuilder.countBuilder(), offset, endOffset); - } - @Override public QuerySqlEntry buildSlice(TableMetadata tableMetadata, SliceVo slice) { final SelectSqlBuilder sqlBuilder = new SelectSqlBuilder(tableMetadata); @@ -77,9 +55,8 @@ public class UnionPrimarySliceQueryStatement implements SliceQueryStatement { ColumnsMetaData primaryKey = tableMetadata.getSinglePrimary(); boolean isDigit = MetaDataUtil.isDigitPrimaryKey(primaryKey); Object offset = translateOffset(isDigit, slice.getBeginIdx()); - Object endOffset = translateOffset(isDigit, slice.getEndIdx()); - sqlBuilder.offset(offset, endOffset); - return new QuerySqlEntry(slice.getTable(), sqlBuilder.builder(), offset, endOffset); + sqlBuilder.inIds(slice.getInIds(), isDigit); + return new QuerySqlEntry(slice.getTable(), sqlBuilder.builder(), offset, offset); } private Object translateOffset(boolean isDigit, String beginIdx) { diff --git a/datachecker-extract/src/main/resources/application-sink.yml b/datachecker-extract/src/main/resources/application-sink.yml index d9348e1ffcf5002668920fe8645d6eba87497021..c798f4f3930b51abd0486832a2bca9af8bb0e2c4 100644 --- a/datachecker-extract/src/main/resources/application-sink.yml +++ b/datachecker-extract/src/main/resources/application-sink.yml @@ -44,4 +44,6 @@ spring: useGlobalDataSourceStat: true connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 maxAllowedPacketSize: 1073741824 + validation-query-timeout: 30 + break-after-acquire-failure: false diff --git a/datachecker-extract/src/main/resources/application-source.yml b/datachecker-extract/src/main/resources/application-source.yml index 37cf04b741446332b773e548fc7933d3c5c38f19..7fd5e54f528c00b480b984313a8f4e795e8b549a 100644 --- a/datachecker-extract/src/main/resources/application-source.yml +++ b/datachecker-extract/src/main/resources/application-source.yml @@ -50,3 +50,5 @@ spring: useGlobalDataSourceStat: true connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 maxAllowedPacketSize: 1073741824 + validation-query-timeout: 30 + break-after-acquire-failure: false \ No newline at end of file diff --git a/datachecker-extract/src/main/resources/mapper/MysqlMetaDataMapper.xml b/datachecker-extract/src/main/resources/mapper/MysqlMetaDataMapper.xml index a22026bfb1da99061911cf1077aae675a7b76460..a25525e080c259c1b10a67bce3df735440d2906e 100644 --- a/datachecker-extract/src/main/resources/mapper/MysqlMetaDataMapper.xml +++ b/datachecker-extract/src/main/resources/mapper/MysqlMetaDataMapper.xml @@ -69,6 +69,9 @@ select count(1) from ${schema}.${name}; + diff --git a/datachecker-extract/src/main/resources/mapper/OpgsMetaDataMapper.xml b/datachecker-extract/src/main/resources/mapper/OpgsMetaDataMapper.xml index c0fdf0aee81e146a4008aab93a554232eae2759a..3cf9f08d169e6bd3233d49594817501d7a749f36 100644 --- a/datachecker-extract/src/main/resources/mapper/OpgsMetaDataMapper.xml +++ b/datachecker-extract/src/main/resources/mapper/OpgsMetaDataMapper.xml @@ -80,6 +80,10 @@ select count(1) from ${schema}.${name}; + + diff --git a/datachecker-extract/src/main/resources/mapper/OracleMetaDataMapper.xml b/datachecker-extract/src/main/resources/mapper/OracleMetaDataMapper.xml index 49047ba292253cc805834e95be5af74c9a6819ab..e023dc4e0cd6a2bfe380ea79c136b78527fa9ee8 100644 --- a/datachecker-extract/src/main/resources/mapper/OracleMetaDataMapper.xml +++ b/datachecker-extract/src/main/resources/mapper/OracleMetaDataMapper.xml @@ -67,6 +67,10 @@ select count(1) from ${schema}.${name} + +