From 871ef28d01ceb3c9193496c1fce12f52375ff80d Mon Sep 17 00:00:00 2001 From: mystarry-sky Date: Thu, 30 Jun 2022 15:41:41 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=BF=81=E7=A7=BB=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 8 + .idea/.gitignore | 8 + README.en.md | 90 +++- README.md | 89 +++- config/application-sink.yml | 23 + config/application-source.yml | 32 ++ config/application.yml | 25 ++ datachecker-check/.gitignore | 33 ++ datachecker-check/pom.xml | 116 +++++ .../check/DatacheckerCheckApplication.java | 18 + .../datachecker/check/cache/Cache.java | 67 +++ .../check/cache/TableStatusRegister.java | 255 +++++++++++ .../check/client/ExtractFeignClient.java | 190 ++++++++ .../client/ExtractFeignClientFallBack.java | 168 +++++++ .../check/client/ExtractSinkFeignClient.java | 17 + .../client/ExtractSourceFeignClient.java | 17 + .../check/client/FeignClientService.java | 189 ++++++++ .../datachecker/check/config/AsyncConfig.java | 38 ++ .../check/config/DataCheckConfig.java | 46 ++ .../check/config/DataCheckProperties.java | 69 +++ .../check/config/DataSourceConfig.java | 29 ++ .../GlobalCheckingExceptionHandler.java | 58 +++ .../check/config/SpringDocConfig.java | 61 +++ .../controller/CheckBlackWhiteController.java | 92 ++++ .../controller/CheckStartController.java | 57 +++ .../IncrementManagerController.java | 43 ++ .../controller/TaskStatusController.java | 57 +++ .../check/modules/bucket/Bucket.java | 73 +++ .../modules/bucket/BuilderBucketHandler.java | 124 ++++++ .../check/AbstractCheckDiffResultBuilder.java | 125 ++++++ .../check/modules/check/CheckDiffResult.java | 47 ++ .../check/modules/check/DataCheckService.java | 54 +++ .../check/modules/check/DataCheckThread.java | 341 ++++++++++++++ .../check/modules/check/DataCheckWapper.java | 40 ++ .../modules/check/ExportCheckResult.java | 22 + .../check/IncrementDataCheckThread.java | 417 ++++++++++++++++++ .../modules/check/QueryRowDataWapper.java | 73 +++ .../check/modules/merkle/MerkleTree.java | 308 +++++++++++++ .../merkle/MerkleTreeDeserializer.java | 81 ++++ .../modules/task/TaskManagerService.java | 33 ++ .../modules/task/TaskManagerServiceImpl.java | 63 +++ .../check/service/CheckBlackWhiteService.java | 105 +++++ .../check/service/CheckService.java | 40 ++ .../check/service/EndpointManagerService.java | 140 ++++++ .../service/IncrementManagerService.java | 46 ++ .../check/service/impl/CheckServiceImpl.java | 248 +++++++++++ .../src/main/resources/application.yml | 45 ++ .../src/main/resources/log4j2.xml | 123 ++++++ .../check/config/DataCheckConfigTest.java | 21 + .../controller/TaskStatusControllerTest.java | 45 ++ .../check/modules/bucket/TestBucket.java | 71 +++ .../check/task/TableStatusRegisterTest.java | 114 +++++ .../DatacheckerCheckApplicationTests.java | 13 + datachecker-common/.gitignore | 33 ++ datachecker-common/pom.xml | 74 ++++ .../common/constant/Constants.java | 12 + .../common/entry/check/DataCheckParam.java | 52 +++ .../common/entry/check/DifferencePair.java | 44 ++ .../datachecker/common/entry/check/Pair.java | 61 +++ .../entry/enums/CheckBlackWhiteMode.java | 40 ++ .../common/entry/enums/CheckMode.java | 32 ++ .../common/entry/enums/ColumnKey.java | 28 ++ .../datachecker/common/entry/enums/DML.java | 35 ++ .../common/entry/enums/DataBaseMeta.java | 24 + .../common/entry/enums/DataBaseType.java | 32 ++ .../common/entry/enums/DataSourceType.java | 23 + .../common/entry/enums/Endpoint.java | 39 ++ .../datachecker/common/entry/enums/IEnum.java | 17 + .../common/entry/enums/ResultEnum.java | 81 ++++ .../common/entry/extract/CheckDiffResult.java | 44 ++ .../common/entry/extract/ColumnsMetaData.java | 40 ++ .../entry/extract/ExtractIncrementTask.java | 33 ++ .../common/entry/extract/ExtractTask.java | 44 ++ .../common/entry/extract/PrimaryMeta.java | 21 + .../common/entry/extract/RowDataHash.java | 25 ++ .../common/entry/extract/SourceDataLog.java | 42 ++ .../common/entry/extract/TableMetadata.java | 39 ++ .../entry/extract/TableMetadataHash.java | 21 + .../common/entry/extract/Topic.java | 24 + .../exception/CheckMetaDataException.java | 16 + .../CheckingAddressConflictException.java | 16 + .../common/exception/CheckingException.java | 20 + .../exception/CheckingPollingException.java | 16 + .../exception/CreateTopicException.java | 32 ++ .../exception/DispatchClientException.java | 25 ++ .../common/exception/ExtractException.java | 17 + .../exception/FeignClientException.java | 19 + .../GlobalCommonExceptionHandler.java | 80 ++++ .../exception/LargeDataDiffException.java | 16 + .../exception/MerkleTreeDepthException.java | 16 + .../exception/ProcessMultipleException.java | 16 + .../exception/TableNotExistException.java | 19 + .../exception/TaskNotFoundException.java | 24 + .../datachecker/common/util/ByteUtil.java | 42 ++ .../datachecker/common/util/EnumUtil.java | 69 +++ .../datachecker/common/util/FileUtils.java | 57 +++ .../datachecker/common/util/HashUtil.java | 49 ++ .../datachecker/common/util/IdWorker.java | 224 ++++++++++ .../common/util/JsonObjectUtil.java | 29 ++ .../datachecker/common/util/SpringUtil.java | 59 +++ .../datachecker/common/util/ThreadUtil.java | 37 ++ .../datachecker/common/web/Result.java | 70 +++ .../src/main/resources/application.properties | 1 + .../DatacheckerCommonApplicationTests.java | 13 + .../datachecker/common/util/HashUtilTest.java | 91 ++++ datachecker-extract/.gitignore | 33 ++ datachecker-extract/pom.xml | 132 ++++++ .../extract/ExtractApplication.java | 17 + .../extract/cache/MetaDataCache.java | 119 +++++ .../cache/TableExtractStatusCache.java | 172 ++++++++ .../extract/client/CheckingFeignClient.java | 56 +++ .../extract/config/AsyncConfig.java | 37 ++ .../extract/config/DruidDataSourceConfig.java | 98 ++++ .../extract/config/ExtractConfig.java | 27 ++ .../extract/config/ExtractProperties.java | 77 ++++ .../config/GlobalExtractExceptionHandler.java | 59 +++ .../extract/config/KafkaConsumerConfig.java | 99 +++++ .../extract/config/KafkaProducerConfig.java | 78 ++++ .../extract/config/SpringDocConfig.java | 73 +++ .../extract/constants/ExtConstants.java | 13 + .../controller/ExtractCleanController.java | 53 +++ .../extract/controller/ExtractController.java | 202 +++++++++ .../controller/ExtractHealthController.java | 19 + .../controller/KafkaManagerController.java | 123 ++++++ .../extract/dao/DataBaseMetaDataDAOImpl.java | 172 ++++++++ .../datachecker/extract/dao/MetaDataDAO.java | 48 ++ .../extract/dao/MetaSqlMapper.java | 72 +++ .../extract/dml/BatchDeleteDmlBuilder.java | 71 +++ .../extract/dml/DeleteDmlBuilder.java | 118 +++++ .../datachecker/extract/dml/DmlBuilder.java | 100 +++++ .../extract/dml/InsertDmlBuilder.java | 74 ++++ .../extract/dml/ReplaceDmlBuilder.java | 75 ++++ .../extract/dml/SelectDmlBuilder.java | 116 +++++ .../extract/service/DataExtractService.java | 116 +++++ .../service/DataExtractServiceImpl.java | 391 ++++++++++++++++ .../extract/service/MetaDataService.java | 113 +++++ .../extract/task/DataManipulationService.java | 290 ++++++++++++ .../extract/task/ExtractTaskBuilder.java | 188 ++++++++ .../extract/task/ExtractTaskThread.java | 126 ++++++ .../extract/task/ExtractThreadSupport.java | 33 ++ .../task/IncrementExtractTaskThread.java | 191 ++++++++ .../task/IncrementExtractThreadSupport.java | 21 + .../extract/task/ResultSetHandler.java | 162 +++++++ .../extract/task/RowDataHashHandler.java | 45 ++ .../extract/task/SelectSqlBulder.java | 178 ++++++++ .../datachecker/extract/util/HashHandler.java | 52 +++ .../extract/util/MetaDataUtil.java | 47 ++ .../src/main/resources/application-sink.yml | 48 ++ .../src/main/resources/application-source.yml | 52 +++ .../src/main/resources/application.yml | 53 +++ .../src/main/resources/log4j2-sink.xml | 141 ++++++ .../src/main/resources/log4j2-source.xml | 143 ++++++ .../extract/ExtractApplicationTests.java | 13 + .../extract/cache/MetaDataCacheTest.java | 33 ++ .../extract/cache/TestByteXOR.java | 61 +++ .../extract/config/DataSourceTest.java | 73 +++ .../dao/DataBaseMetaDataDAOImplTests.java | 37 ++ .../extract/dao/enums/EnumTest.java | 27 ++ .../extract/service/MetaDataServiceTest.java | 24 + .../extract/task/ExtractTaskBuilderTest.java | 36 ++ .../extract/task/SelectSqlBulderTest.java | 112 +++++ datachecker-mock-data/.gitignore | 33 ++ datachecker-mock-data/pom.xml | 80 ++++ .../extract/MockDataApplication.java | 13 + .../datachecker/extract/TableRowCount.java | 17 + .../extract/config/AsyncConfig.java | 37 ++ .../extract/config/DruidDataSourceConfig.java | 46 ++ .../extract/config/ExtractConfig.java | 25 ++ .../extract/config/SpringDocConfig.java | 60 +++ .../controller/ExtractMockController.java | 66 +++ .../service/ExtractMockDataService.java | 53 +++ .../service/ExtractMockTableService.java | 103 +++++ .../service/thread/ExtractMockDataThread.java | 134 ++++++ .../src/main/resources/application.yml | 40 ++ .../src/main/resources/log4j2.xml | 89 ++++ openGauss-tools-datachecker-performance.iml | 12 + pom.xml | 110 +++++ 177 files changed, 12779 insertions(+), 18 deletions(-) create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 config/application-sink.yml create mode 100644 config/application-source.yml create mode 100644 config/application.yml create mode 100644 datachecker-check/.gitignore create mode 100644 datachecker-check/pom.xml create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/DatacheckerCheckApplication.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/cache/Cache.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/cache/TableStatusRegister.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractFeignClient.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractFeignClientFallBack.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractSinkFeignClient.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractSourceFeignClient.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/client/FeignClientService.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/config/AsyncConfig.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataCheckConfig.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataCheckProperties.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataSourceConfig.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/config/GlobalCheckingExceptionHandler.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/config/SpringDocConfig.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/CheckBlackWhiteController.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/CheckStartController.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/IncrementManagerController.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/TaskStatusController.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/bucket/Bucket.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/bucket/BuilderBucketHandler.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/AbstractCheckDiffResultBuilder.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckDiffResult.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckService.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckThread.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckWapper.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/ExportCheckResult.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/IncrementDataCheckThread.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/QueryRowDataWapper.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/merkle/MerkleTree.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/merkle/MerkleTreeDeserializer.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/task/TaskManagerService.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/task/TaskManagerServiceImpl.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckBlackWhiteService.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckService.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/service/EndpointManagerService.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/service/IncrementManagerService.java create mode 100644 datachecker-check/src/main/java/org/opengauss/datachecker/check/service/impl/CheckServiceImpl.java create mode 100644 datachecker-check/src/main/resources/application.yml create mode 100644 datachecker-check/src/main/resources/log4j2.xml create mode 100644 datachecker-check/src/test/java/org/opengauss/datachecker/check/config/DataCheckConfigTest.java create mode 100644 datachecker-check/src/test/java/org/opengauss/datachecker/check/controller/TaskStatusControllerTest.java create mode 100644 datachecker-check/src/test/java/org/opengauss/datachecker/check/modules/bucket/TestBucket.java create mode 100644 datachecker-check/src/test/java/org/opengauss/datachecker/check/task/TableStatusRegisterTest.java create mode 100644 datachecker-check/src/test/java/org/opengauss/datacheckercheck/DatacheckerCheckApplicationTests.java create mode 100644 datachecker-common/.gitignore create mode 100644 datachecker-common/pom.xml create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/Constants.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/DataCheckParam.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/DifferencePair.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/Pair.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/CheckBlackWhiteMode.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/CheckMode.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/ColumnKey.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DML.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataBaseMeta.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataBaseType.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataSourceType.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/Endpoint.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/IEnum.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/ResultEnum.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/CheckDiffResult.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ColumnsMetaData.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractIncrementTask.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractTask.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/PrimaryMeta.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/RowDataHash.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SourceDataLog.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadata.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadataHash.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/Topic.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckMetaDataException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingAddressConflictException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingPollingException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CreateTopicException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/DispatchClientException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/ExtractException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/FeignClientException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/GlobalCommonExceptionHandler.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/LargeDataDiffException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/MerkleTreeDepthException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/ProcessMultipleException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/TableNotExistException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/TaskNotFoundException.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ByteUtil.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/util/EnumUtil.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/util/FileUtils.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/util/HashUtil.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/util/IdWorker.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/util/JsonObjectUtil.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/util/SpringUtil.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ThreadUtil.java create mode 100644 datachecker-common/src/main/java/org/opengauss/datachecker/common/web/Result.java create mode 100644 datachecker-common/src/main/resources/application.properties create mode 100644 datachecker-common/src/test/java/org/opengauss/datachecker/common/DatacheckerCommonApplicationTests.java create mode 100644 datachecker-common/src/test/java/org/opengauss/datachecker/common/util/HashUtilTest.java create mode 100644 datachecker-extract/.gitignore create mode 100644 datachecker-extract/pom.xml create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/ExtractApplication.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/MetaDataCache.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/TableExtractStatusCache.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/client/CheckingFeignClient.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/AsyncConfig.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/DruidDataSourceConfig.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/ExtractConfig.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/ExtractProperties.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/GlobalExtractExceptionHandler.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/KafkaConsumerConfig.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/KafkaProducerConfig.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/SpringDocConfig.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/constants/ExtConstants.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractCleanController.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractController.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractHealthController.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/KafkaManagerController.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/DataBaseMetaDataDAOImpl.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/MetaDataDAO.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/MetaSqlMapper.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/BatchDeleteDmlBuilder.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/DeleteDmlBuilder.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/DmlBuilder.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/InsertDmlBuilder.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/ReplaceDmlBuilder.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/SelectDmlBuilder.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractService.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractServiceImpl.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/DataManipulationService.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractTaskBuilder.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractTaskThread.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractThreadSupport.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/IncrementExtractTaskThread.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/IncrementExtractThreadSupport.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ResultSetHandler.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/RowDataHashHandler.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/SelectSqlBulder.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/HashHandler.java create mode 100644 datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/MetaDataUtil.java create mode 100644 datachecker-extract/src/main/resources/application-sink.yml create mode 100644 datachecker-extract/src/main/resources/application-source.yml create mode 100644 datachecker-extract/src/main/resources/application.yml create mode 100644 datachecker-extract/src/main/resources/log4j2-sink.xml create mode 100644 datachecker-extract/src/main/resources/log4j2-source.xml create mode 100644 datachecker-extract/src/test/java/org/opengauss/datachecker/extract/ExtractApplicationTests.java create mode 100644 datachecker-extract/src/test/java/org/opengauss/datachecker/extract/cache/MetaDataCacheTest.java create mode 100644 datachecker-extract/src/test/java/org/opengauss/datachecker/extract/cache/TestByteXOR.java create mode 100644 datachecker-extract/src/test/java/org/opengauss/datachecker/extract/config/DataSourceTest.java create mode 100644 datachecker-extract/src/test/java/org/opengauss/datachecker/extract/dao/DataBaseMetaDataDAOImplTests.java create mode 100644 datachecker-extract/src/test/java/org/opengauss/datachecker/extract/dao/enums/EnumTest.java create mode 100644 datachecker-extract/src/test/java/org/opengauss/datachecker/extract/service/MetaDataServiceTest.java create mode 100644 datachecker-extract/src/test/java/org/opengauss/datachecker/extract/task/ExtractTaskBuilderTest.java create mode 100644 datachecker-extract/src/test/java/org/opengauss/datachecker/extract/task/SelectSqlBulderTest.java create mode 100644 datachecker-mock-data/.gitignore create mode 100644 datachecker-mock-data/pom.xml create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/MockDataApplication.java create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/TableRowCount.java create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/AsyncConfig.java create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/DruidDataSourceConfig.java create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/ExtractConfig.java create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/SpringDocConfig.java create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/controller/ExtractMockController.java create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/ExtractMockDataService.java create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/ExtractMockTableService.java create mode 100644 datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/thread/ExtractMockDataThread.java create mode 100644 datachecker-mock-data/src/main/resources/application.yml create mode 100644 datachecker-mock-data/src/main/resources/log4j2.xml create mode 100644 openGauss-tools-datachecker-performance.iml create mode 100644 pom.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a6920db --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +/.idea +/datachecker-check/.mvn +.mvn +/logs +/datachecker-check/Result +/*/mvnw +/*/mvnw.cmd +/*/logs diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..df73d40 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/../../../../../:\code\tool\openGauss-tools-datachecker-performance\.idea/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/README.en.md b/README.en.md index a91e6a9..f96cdda 100644 --- a/README.en.md +++ b/README.en.md @@ -1,22 +1,96 @@ # openGauss-tools-datachecker-performance #### Description -{**When you're done, you can delete the content in this README and update the file with details for others getting started with your repository**} +Opengauss data migration verification tool, including full data verification and incremental data verification. #### Software Architecture -Software architecture description +Full data verification: JDBC is used to extract the source and target data, and the extraction results are temporarily stored in Kafka. The verification service obtains the extraction results of the specified table from Kafka through the extraction service for verification. Finally, output the verification results to the file in the specified path. + + + +Incremental data verification, through debezium monitoring the data change records of the source database, the extraction service regularly processes the change records of debezium according to a certain frequency, and makes statistics on the change records. Send the statistical results to the data verification service. The data verification service initiates incremental data verification and outputs the verification results to the specified path file. #### Installation -1. xxxx -2. xxxx -3. xxxx +1. Obtain the data verification service jar package and the configuration file template (datachecker-check.jar/datachecker-extract.jar, application.yml, application sink.yml, application source.yml) +2. Copy the jar package and configuration file to the specified server directory, configure the relevant configuration file, and start the corresponding jar service. + +3. Download and start Kafka #### Instructions -1. xxxx -2. xxxx -3. xxxx +**Start zookeeper** + +``` +cd {path}/kafka_2.12-3.1.1/bin +``` + +Start the ZooKeeper service + +Note: Soon, ZooKeeper will no longer be required by Apache Kafka. + +``` +bin/zookeeper-server-start.sh config/zookeeper.properties +``` + +Open another terminal session and run: + +``` +sh bin/zookeeper-server-start.sh -daemon config/zookeeper.properties +``` + +**Start Kafka** + +Start the Kafka broker service + +``` +bin/kafka-server-start.sh config/server.properties + +sh bin/kafka-server-start.sh -daemon /config/server.properties +``` + +**Start Kafka interface service Kafka Eagle (optional)** + +``` +cd {path}/kafka-eagle/bin +sh ke.sh start | restart +``` + +* Welcome, Now you can visit http://ip:port + +* Account:admin ,Password:123456(here is the default account and password of Kafka Eagle) + +**Start datachecker performance service** + +``` +Source side extraction service +java -jar datachecker-extract.jar -Dspring.config.additional-location=.\config\application-source.yml + +Destination extraction service +java -jar datachecker-extract.jar -Dspring.config.additional-location=.\config\application-sink.yml + +check service +java -jar datachecker-check.jar -Dspring.config.additional-location=.\config\application.yml +``` + + + +**Developer local startup service** + +Add virtual machine parameter VM option in startup configuration: + +``` +Source side extraction service +-Dspring.config.additional-location=.\config\application-source.yml + +Destination extraction service +-Dspring.config.additional-location=.\config\application-sink.yml + +check service +-Dspring.config.additional-location=.\config\application.yml +``` + + #### Contribution diff --git a/README.md b/README.md index 9d6e3df..46c723c 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,94 @@ # openGauss-tools-datachecker-performance #### 介绍 -{**以下是 Gitee 平台说明，您可以替换此简介** -Gitee 是 OSCHINA 推出的基于 Git 的代码托管平台（同时支持 SVN）。专为开发者提供稳定、高效、安全的云端软件开发协作平台 -无论是个人、团队、或是企业，都能够用 Gitee 实现代码托管、项目管理、协作开发。企业项目请看 [https://gitee.com/enterprises](https://gitee.com/enterprises)} +openGauss数据迁移校验工具 ，包含全量数据校验以及增量数据校验。 #### 软件架构 -软件架构说明 +全量数据校验，采用JDBC方式抽取源端和目标端数据，并将抽取结果暂存到kafka中，校验服务通过抽取服务从kafka中获取指定表的抽取结果，进行校验。最后将校验结果输出到指定路径的文件文件中。 + +增量数据校验，通过debezium监控源端数据库的数据变更记录，抽取服务按照一定的频率定期处理debezium的变更记录，对变更记录进行统计。将统计结果发送给数据校验服务。由数据校验服务发起增量数据校验，并将校验结果输出到指定路径文件。 #### 安装教程 -1. xxxx -2. xxxx -3. xxxx +1. 获取数据校验服务jar包，及配置文件模版（datachecker-check.jar/datachecker-extract.jar,application.yml,application-sink.yml,application-source.yml） +2. 将jar包以及配置文件copy到指定服务器目录，并配置相关配置文件，启动相应的jar服务即可。 +3. 下载并启动kafka #### 使用说明 -1. xxxx -2. xxxx -3. xxxx +**启动Zookeeper** + +``` +cd {path}/kafka_2.12-3.1.1/bin +``` + +Start the ZooKeeper service + +Note: Soon, ZooKeeper will no longer be required by Apache Kafka. + +``` +bin/zookeeper-server-start.sh config/zookeeper.properties +``` + +Open another terminal session and run: + +``` +sh bin/zookeeper-server-start.sh -daemon config/zookeeper.properties +``` + +**启动Kafka** + +Start the Kafka broker service + +``` +bin/kafka-server-start.sh config/server.properties + +sh bin/kafka-server-start.sh -daemon /config/server.properties +``` + +**启动kafka界面服务 kafka-eagle（可选项）** + +``` +cd {path}/kafka-eagle/bin +sh ke.sh start | restart +``` + +* Welcome, Now you can visit http://ip:port + +* Account:admin ,Password:123456 (这里是kafka-eagle默认账户和密码) + +**启动数据校验服务** + +``` +源端抽取服务 +java -jar datachecker-extract.jar -Dspring.config.additional-location=.\config\application-source.yml + +宿端抽取服务 +java -jar datachecker-extract.jar -Dspring.config.additional-location=.\config\application-sink.yml + +校验服务 +java -jar datachecker-check.jar -Dspring.config.additional-location=.\config\application.yml +``` + + + +**开发人员本地 启动服务** + +在启动配置中添加虚拟机参数 VM Option : + +``` +源端抽取服务 +-Dspring.config.additional-location=.\config\application-source.yml + +宿端抽取服务 +-Dspring.config.additional-location=.\config\application-sink.yml + +校验服务 +-Dspring.config.additional-location=.\config\application.yml +``` + + #### 参与贡献 diff --git a/config/application-sink.yml b/config/application-sink.yml new file mode 100644 index 0000000..27578aa --- /dev/null +++ b/config/application-sink.yml @@ -0,0 +1,23 @@ +server: + port: 9001 + +debug: false + +spring: + check: + server-uri: http://{ip}:{port} # 数据校验服务地址 + extract: + # 前置：源端查询表mate数据时，获取单表数据总记录数， query-table-row-count配置是为控制流程是否通过查询表数据方式获取当前表数据总数。 + query-table-row-count: false + schema: jack # 宿端opengauss 用于校验数据schema + databaseType: OG # 宿端数据库类型 OG opengauss + kafka: + bootstrap-servers: { ip }:{port};{ip}:{port} # kafka 集群地址 + datasource: + druid: + dataSourceOne: + driver-class-name: org.postgresql.Driver + # 宿端opengauss用于校验数据库链接地址 + url: jdbc:postgresql://{ip}:{port}/{schema_db}?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC + username: # 宿端opengauss用于校验的用户名称 + password: # 宿端opengauss用于校验的用户名称密码 \ No newline at end of file diff --git a/config/application-source.yml b/config/application-source.yml new file mode 100644 index 0000000..ebf196d --- /dev/null +++ b/config/application-source.yml @@ -0,0 +1,32 @@ +server: + port: 9002 + +debug: false + +spring: + check: + server-uri: http://{ip}:{port} # 数据校验服务地址 + extract: + query-table-row-count: false # 前置：源端查询表mate数据时，获取单表数据总记录数， query-table-row-count配置是为控制流程是否通过查询表数据方式获取当前表数据总数。 + schema: test # 源端数据实例 + databaseType: MS # 源端数据库类型 MS mysql + debezium-topic: my_oracle_connector_002 # debezium监听表增量数据，使用单一topic进行增量数据管理 + debezium-groupId: debezium-extract-group # d debezium增量迁移topic ，groupId消费Group设置 + debezium-topic-partitions: 1 # debezium监听topic 分区数量配置 + debezium-tables: # debezium-tables配置debezium监听的表名称列表; 该配置只在源端服务配置并生效 + TEST1, + TEST5, + TEST3 + debezium-enable: true #是否开启增量debezium配置 默认不开启 + debezium-time-period: 1 # debezium增量迁移校验 时间周期 24*60 单位分钟 + debezium-num-period: 1000 #debezium增量迁移校验 统计增量变更记录数量阀值，默认值1000 阀值应大于100 + + kafka: + bootstrap-servers: { ip }:{port};{ip}:{port} # kafka 集群地址 + datasource: + druid: + dataSourceOne: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://{ip}:{port}/{database}?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC&allowPublicKeyRetrieval=true + username: # 源端mysql用于校验的用户名称 + password: # 源端mysql用于校验的用户名称密码 \ No newline at end of file diff --git a/config/application.yml b/config/application.yml new file mode 100644 index 0000000..dae20b3 --- /dev/null +++ b/config/application.yml @@ -0,0 +1,25 @@ +server: + port: 9000 + +debug: false + +spring: + datasource: + druid: + dataCheck: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://{ip}:{port}/check?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC&allowPublicKeyRetrieval=true + username: # 数据库用户名称 + password: # 数据库用户访问密码 +data: + check: + #data-path: local_path/xxx # 配置数据校验结果输出本地路径 + data-path: D:\code\tool\datachecker-tool\datachecker-check + bucket-expect-capacity: 10 # 桶容量范围最小值为1 + source-uri: http://{ip}:{port} # 配置源端服务地址和服务端口server.port + sink-uri: http://{ip}:{port} # 配置源端服务地址和服务端口server.port + black-white-mode: BLACK #大写 配置校验表黑白名单启动模式 【BLACK 启动黑名单校验，WHITE 启动白名单校验】 + + + + diff --git a/datachecker-check/.gitignore b/datachecker-check/.gitignore new file mode 100644 index 0000000..549e00a --- /dev/null +++ b/datachecker-check/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/datachecker-check/pom.xml b/datachecker-check/pom.xml new file mode 100644 index 0000000..5247680 --- /dev/null +++ b/datachecker-check/pom.xml @@ -0,0 +1,116 @@ + + + 4.0.0 + + org.opengauss + openGauss-tools-datachecker-performance + 0.0.1 + ../pom.xml + + + datachecker-check + 0.0.1 + datachecker-check + jar + datachecker-check + + 11 + 0.0.1 + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.opengauss + datachecker-common + ${datachecker.version} + + + org.springframework.boot + spring-boot-starter-log4j2 + + + org.springframework.boot + spring-boot-starter-data-jdbc + + + + mysql + mysql-connector-java + provided + + + com.alibaba + druid + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.projectlombok + lombok + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + org.springdoc + springdoc-openapi-ui + + + org.springframework.boot + spring-boot-starter-validation + + + com.alibaba + fastjson + + + org.apache.commons + commons-collections4 + + + com.google.guava + guava + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + mysql + mysql-connector-java + + + + + + + + diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/DatacheckerCheckApplication.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/DatacheckerCheckApplication.java new file mode 100644 index 0000000..4ec9eb7 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/DatacheckerCheckApplication.java @@ -0,0 +1,18 @@ +package org.opengauss.datachecker.check; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableAsync; + + +@EnableAsync +@EnableFeignClients(basePackages = {"org.opengauss.datachecker.check.client"}) +@SpringBootApplication +public class DatacheckerCheckApplication { + + public static void main(String[] args) { + SpringApplication.run(DatacheckerCheckApplication.class, args); + } + +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/cache/Cache.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/cache/Cache.java new file mode 100644 index 0000000..7d9117b --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/cache/Cache.java @@ -0,0 +1,67 @@ +package org.opengauss.datachecker.check.cache; + +import java.util.Set; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/25 + * @since ：11 + */ +public interface Cache { + + /** + * 初始化缓存 并给键值设置默认值 + * + * @param keys 缓存键值 + */ + void init(Set keys); + + /** + * 添加键值对到缓存 + * + * @param key 键 + * @param value 值 + */ + void put(K key, V value); + + /** + * 根据key查询缓存 + * + * @param key 缓存key + * @return 缓存value + */ + V get(K key); + + /** + * 获取缓存Key集合 + * + * @return Key集合 + */ + Set getKeys(); + + /** + * 更新缓存数据 + * + * @param key 缓存key + * @param value 缓存value + * @return 更新后的缓存value + */ + V update(K key, V value); + + /** + * 删除指定key缓存 + * + * @param key key + */ + void remove(K key); + + /** + * 清除全部缓存 + */ + void removeAll(); + + /** + * 缓存持久化接口 将缓存信息持久化到本地 + */ + void persistent(); +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/cache/TableStatusRegister.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/cache/TableStatusRegister.java new file mode 100644 index 0000000..0a743c5 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/cache/TableStatusRegister.java @@ -0,0 +1,255 @@ +package org.opengauss.datachecker.check.cache; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.exception.ExtractException; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import javax.validation.constraints.NotEmpty; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.*; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/25 + * @since ：11 + */ +@Slf4j +@Service +public class TableStatusRegister implements Cache { + + /** + * 任务状态缓存默认值 + */ + private static final int TASK_STATUS_DEFAULT_VALUE = 0; + /** + * 任务状态 源端 和宿端均完成数据抽取 + */ + public static final int TASK_STATUS_COMPLATED_VALUE = 3; + /** + * 任务状态 校验服务已进行当前任务校验 + */ + public static final int TASK_STATUS_COMSUMER_VALUE = 7; + /** + * 状态自检线程名称 + */ + private static final String SELF_CHECK_THREAD_NAME = "task-register-self-check-thread"; + + /** + * 数据抽取任务对应表 执行状态缓存 + * {@code tableStatusCache} : key 为数据抽取表名称 + * {@code tableStatusCache} : value 为数据抽取表完成状态 + * value 值初始化状态为 0 + * 源端完成表识为 1 则更新当前表缓存状态为 value = value | 1 + * 宿端完成表识为 2 则更新当前表缓存状态为 value = value | 2 + * 数据校验标识为 4 则更新当前表缓存状态为 value = value | 4 + */ + private static final Map TABLE_STATUS_CACHE = new ConcurrentHashMap<>(); + + /** + * 单线程定时任务 + */ + private static final ScheduledExecutorService SCHEDULED_EXECUTOR = Executors.newSingleThreadScheduledExecutor(); + /** + * 完成表集合 + */ + private static final Set COMPLATED_TABLE = new HashSet<>(); + /** + * {@code complatedTableQueue} poll消费记录 + */ + private static final Set CONSUMER_COMPLATED_TABLE = new HashSet<>(); + + /** + * + */ + private static final BlockingDeque COMPLATED_TABLE_QUEUE = new LinkedBlockingDeque<>(); + + /** + * 服务启动恢复缓存信息。根据持久化缓存数据，恢复历史数据 + * 扫描指定位置的缓存文件，解析JSON字符串反序列化当前缓存数据 + */ + @PostConstruct + public void recover() { + selfCheck(); + // 扫描指定位置的缓存文件，解析JSON字符串反序列化当前缓存数据 + } + + /** + * 开启并执行自检线程 + */ + public void selfCheck() { + SCHEDULED_EXECUTOR.scheduleWithFixedDelay(() -> { + Thread.currentThread().setName(SELF_CHECK_THREAD_NAME); + doCheckingStatus(); + + }, 0, 1, TimeUnit.SECONDS); + } + + /** + * 完成数据抽取任务数量 和 已消费的完成任务数量一致，且大于0时，认为本次校验服务已完成 + * + * @return + */ + public boolean isCheckComplated() { + return CONSUMER_COMPLATED_TABLE.size() > 0 && CONSUMER_COMPLATED_TABLE.size() == COMPLATED_TABLE.size(); + } + + /** + * 增量任务状态重置 + */ + public void rest() { + CONSUMER_COMPLATED_TABLE.clear(); + COMPLATED_TABLE.clear(); + init(TABLE_STATUS_CACHE.keySet()); + } + + public int complateSize() { + return COMPLATED_TABLE.size(); + } + + public boolean isEmpty() { + return CONSUMER_COMPLATED_TABLE.size() == 0 && COMPLATED_TABLE.size() == 0; + } + + public boolean hasExtractComplated() { + return COMPLATED_TABLE.size() > 0; + } + + /** + * 初始化缓存 并给键值设置默认值 + * + * @param keys + * @return + */ + @Override + public void init(@NotEmpty Set keys) { + keys.forEach(key -> { + TABLE_STATUS_CACHE.put(key, TASK_STATUS_DEFAULT_VALUE); + }); + } + + /** + * 添加表状态对到缓存 + * + * @param key 键 + * @param value 值 + * @return 返回任务状态 + */ + @Override + public void put(String key, Integer value) { + if (TABLE_STATUS_CACHE.containsKey(key)) { + // 当前key已存在不能重复添加 + throw new ExtractException("The current key= " + key + " already exists and cannot be added repeatedly"); + } + TABLE_STATUS_CACHE.put(key, value); + } + + /** + * 根据key查询缓存 + * + * @param key 缓存key + * @return 缓存value + */ + @Override + public Integer get(String key) { + return TABLE_STATUS_CACHE.getOrDefault(key, -1); + } + + /** + * 获取缓存Key集合 + * + * @return Key集合 + */ + @Override + public Set getKeys() { + return TABLE_STATUS_CACHE.keySet(); + } + + /** + * 更新缓存数据 + * + * @param key 缓存key + * @param value 缓存value + * @return 更新后的缓存value + */ + @Override + public Integer update(String key, Integer value) { + if (!TABLE_STATUS_CACHE.containsKey(key)) { + log.error("current key={} does not exist", key); + return 0; + } + Integer odlValue = TABLE_STATUS_CACHE.get(key); + TABLE_STATUS_CACHE.put(key, odlValue | value); + return TABLE_STATUS_CACHE.get(key); + } + + + /** + * 删除指定key缓存 + * + * @param key key + */ + @Override + public void remove(String key) { + TABLE_STATUS_CACHE.remove(key); + } + + /** + * 清除全部缓存 + */ + @Override + public void removeAll() { + COMPLATED_TABLE.clear(); + CONSUMER_COMPLATED_TABLE.clear(); + TABLE_STATUS_CACHE.clear(); + COMPLATED_TABLE_QUEUE.clear(); + log.info("table status register cache information clearing"); + } + + /** + * 缓存持久化接口 将缓存信息持久化到本地 + * 将缓存信息持久化到本地的缓存文件，序列化为JSON字符串，保存到本地指定文件中 + */ + @Override + public void persistent() { + } + + /** + * 返回并删除 已完成数据抽取任务的统计队列{@code complatedTableQueue} 头节点， + * 如果队列为空，则返回{@code null} + * + * @return 返回队列头节点，如果队列为空，则返回{@code null} + */ + public String complatedTablePoll() { + return COMPLATED_TABLE_QUEUE.poll(); + } + + /** + * 检查是否存在已完成数据抽取任务。若已完成，则返回true + * + * @return true 有已完成数据抽取的任务 + */ + private void doCheckingStatus() { + Set keys = TABLE_STATUS_CACHE.keySet(); + keys.forEach(tableName -> { + final int taskStatus = TABLE_STATUS_CACHE.get(tableName); + log.debug("check table=[{}] status=[{}] ", tableName, taskStatus); + if (!COMPLATED_TABLE.contains(tableName)) { + if (taskStatus == TableStatusRegister.TASK_STATUS_COMPLATED_VALUE) { + COMPLATED_TABLE.add(tableName); + COMPLATED_TABLE_QUEUE.add(tableName); + log.info("extract [{}] complated", tableName); + } + } + + if (!CONSUMER_COMPLATED_TABLE.contains(tableName)) { + if (taskStatus == TableStatusRegister.TASK_STATUS_COMSUMER_VALUE) { + CONSUMER_COMPLATED_TABLE.add(tableName); + log.info("consumer [{}] complated", tableName); + } + } + }); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractFeignClient.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractFeignClient.java new file mode 100644 index 0000000..38897e7 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractFeignClient.java @@ -0,0 +1,190 @@ +package org.opengauss.datachecker.check.client; + +import org.opengauss.datachecker.common.entry.check.IncrementCheckConifg; +import org.opengauss.datachecker.common.entry.enums.CheckBlackWhiteMode; +import org.opengauss.datachecker.common.entry.enums.DML; +import org.opengauss.datachecker.common.entry.extract.*; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +public interface ExtractFeignClient { + /** + * 服务健康检查 + * + * @return 返回接口相应结果 + */ + @GetMapping("/extract/health") + Result health(); + + /** + * 端点加载元数据信息 + * + * @return 返回元数据 + */ + @GetMapping("/extract/load/database/meta/data") + Result> queryMetaDataOfSchema(); + + /** + * 抽取任务构建 + * + * @param processNo 执行进程编号 + * @return 返回构建任务集合 + */ + @PostMapping("/extract/build/task/all") + Result> buildExtractTaskAllTables(@RequestParam(name = "processNo") String processNo); + + /** + * 宿端抽取任务配置 + * + * @param processNo 执行进程编号 + * @param taskList 源端任务列表 + * @return 请求结果 + */ + @PostMapping("/extract/config/sink/task/all") + Result buildExtractTaskAllTables(@RequestParam(name = "processNo") String processNo, + @RequestBody List taskList); + + /** + * 全量抽取业务处理流程 + * + * @param processNo 执行进程序号 + * @return 执行结果 + */ + @PostMapping("/extract/exec/task/all") + Result execExtractTaskAllTables(@RequestParam(name = "processNo") String processNo); + + + /** + * 查询指定表对应Topic信息 + * + * @param tableName 表名称 + * @return Topic信息 + */ + @GetMapping("/extract/topic/info") + Result queryTopicInfo(@RequestParam(name = "tableName") String tableName); + + /** + * 获取增量 Topic信息 + * + * @param tableName 表名称 + * @return 返回表对应的Topic信息 + */ + @GetMapping("/extract/increment/topic/info") + Result getIncrementTopicInfo(@RequestParam(name = "tableName") String tableName); + + /** + * 查询指定topic数据 + * + * @param tableName 表名称 + * @param partitions topic分区 + * @return topic数据 + */ + @GetMapping("/extract/query/topic/data") + Result> queryTopicData(@RequestParam("tableName") String tableName, + @RequestParam("partitions") int partitions); + + /** + * 查询指定增量topic数据 + * + * @param tableName 表名称 + * @return topic数据 + */ + @GetMapping("/extract/query/increment/topic/data") + Result> queryIncrementTopicData(@RequestParam("tableName") String tableName); + + /** + * 清理对端环境 + * + * @param processNo 执行进程序号 + * @return 执行结果 + */ + @PostMapping("/extract/clean/environment") + Result cleanEnvironment(@RequestParam(name = "processNo") String processNo); + + /** + * 清除抽取端 任务缓存 + * + * @return 执行结果 + */ + @PostMapping("/extract/clean/task") + Result cleanTask(); + + /** + * 根据参数构建修复语句 + * + * @param schema 待修复端DB对应schema + * @param tableName 表名称 + * @param dml 修复类型{@link DML} + * @param diffSet 差异主键集合 + * @return 返回修复语句集合 + */ + @PostMapping("/extract/build/repairDML") + Result> buildRepairDml(@RequestParam(name = "schema") String schema, + @RequestParam(name = "tableName") String tableName, + @RequestParam(name = "dml") DML dml, + @RequestBody Set diffSet); + + /** + * 下发增量日志数据 + * + * @param dataLogList 增量数据日志 + */ + @PostMapping("/extract/increment/logs/data") + void notifyIncrementDataLogs(List dataLogList); + + /** + * 查询表元数据哈希信息 + * + * @param tableName 表名称 + * @return 表元数据哈希 + */ + @PostMapping("/extract/query/table/metadata/hash") + Result queryTableMetadataHash(@RequestParam(name = "tableName") String tableName); + + /** + * 提取增量日志数据记录 + * + * @param dataLog 日志记录 + * @return 返回抽取结果 + */ + @PostMapping("/extract/query/secondary/data/row/hash") + Result> querySecondaryCheckRowData(@RequestBody SourceDataLog dataLog); + + /** + * 查询抽取端数据库schema信息 + * + * @return 返回schema + */ + @GetMapping("/extract/query/database/schema") + Result getDatabaseSchema(); + + /** + * 更新黑白名单列表 + * + * @param mode 黑白名单模式枚举{@linkplain CheckBlackWhiteMode} + * @param tableList 黑白名单列表-表名称集合 + */ + @PostMapping("/extract/refush/black/white/list") + void refushBlackWhiteList(@RequestParam CheckBlackWhiteMode mode, @RequestBody List tableList); + + /** + * 配置增量校验场景 debezium相关配置信息 + * + * @param conifg debezium相关配置 + * @return 返回请求结果 + */ + @PostMapping("/extract/debezium/topic/config") + Result configIncrementCheckEnvironment(IncrementCheckConifg conifg); +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractFeignClientFallBack.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractFeignClientFallBack.java new file mode 100644 index 0000000..d7db0fe --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractFeignClientFallBack.java @@ -0,0 +1,168 @@ +//package org.opengauss.datachecker.check.client; +// +//import lombok.extern.slf4j.Slf4j; +//import org.opengauss.datachecker.common.entry.check.IncrementCheckConifg; +//import org.opengauss.datachecker.common.entry.enums.CheckBlackWhiteMode; +//import org.opengauss.datachecker.common.entry.enums.DML; +//import org.opengauss.datachecker.common.entry.extract.*; +//import org.opengauss.datachecker.common.web.Result; +// +//import java.util.List; +//import java.util.Map; +//import java.util.Set; +// +///** +// * @author ：wangchao +// * @date ：Created in 2022/5/30 +// * @since ：11 +// */ +//@Slf4j +//public class ExtractFeignClientFallBack { +// public ExtractFeignClient getClient(Throwable throwable) { +// return new ExtractFeignClient() { +// /** +// * 服务健康检查 +// * +// * @return 返回接口相应结果 +// */ +// @Override +// public Result health() { +// log.error("health check error：{}", throwable.getMessage()); +// return Result.error("health check error"); +// } +// +// /** +// * 端点加载元数据信息 +// * +// * @return 返回元数据 +// */ +// @Override +// public Result> queryMetaDataOfSchema() { +// log.error("query database metadata error ：{}", throwable.getMessage()); +// return Result.error("query database metadata error"); +// } +// +// /** +// * 抽取任务构建 +// * +// * @param processNo 执行进程编号 +// */ +// @Override +// public Result> buildExtractTaskAllTables(String processNo) { +// log.error("build extract task error , process number ={} ：{}", processNo, throwable.getMessage()); +// return Result.error(String.format("build extract task error : process number =%s", processNo)); +// } +// +// /** +// * 宿端抽取任务配置 +// * +// * @param processNo 执行进程编号 +// * @param taskList 源端任务列表 +// * @return 请求结果 +// */ +// @Override +// public Result buildExtractTaskAllTables(String processNo, List taskList) { +// return null; +// } +// +// /** +// * 全量抽取业务处理流程 +// * +// * @param processNo 执行进程序号 +// * @return 执行结果 +// */ +// @Override +// public Result execExtractTaskAllTables(String processNo) { +// log.error("runing extract task error , process number ={} ：{}", processNo, throwable.getMessage()); +// return Result.error(String.format("runing extract task error : process number =%s", processNo)); +// } +// +// +// @Override +// public Result queryTopicInfo(String tableName) { +// return null; +// } +// +// @Override +// public Result getIncrementTopicInfo(String tableName) { +// return null; +// } +// +// /** +// * 查询指定topic数据 +// * @param tableName topic名称 +// * @param partitions topic分区 +// * @return topic数据 +// */ +// @Override +// public Result> queryTopicData(String tableName, int partitions) { +// return null; +// } +// +// /** +// * 查询指定增量topic数据 +// * +// * @param tableName 表名称 +// * @return topic数据 +// */ +// @Override +// public Result> queryIncrementTopicData(String tableName) { +// return null; +// } +// +// /** +// * 清理对端环境 +// * +// * @param processNo 执行进程序号 +// * @return 执行结果 +// */ +// @Override +// public Result cleanEnvironment(String processNo) { +// log.error("clean environment error , process number ={} : ", processNo, throwable); +// return null; +// } +// +// @Override +// public Result cleanTask() { +// return null; +// } +// +// @Override +// public Result> buildRepairDml(String schema, String tableName, DML dml, Set diffSet) { +// log.error("build Repair DML error , tableName=[{}] dml=[{}] diffs=[{}] :", tableName, dml.getDescription(), diffSet, throwable); +// return null; +// } +// +// @Override +// public void notifyIncrementDataLogs(List dataLogList) { +// +// } +// +// @Override +// public Result queryTableMetadataHash(String tableName) { +// return null; +// } +// +// @Override +// public Result> querySecondaryCheckRowData(SourceDataLog dataLog) { +// return null; +// } +// +// @Override +// public Result getDatabaseSchema() { +// return null; +// } +// +// @Override +// public void refushBlackWhiteList(CheckBlackWhiteMode mode, List whiteList) { +// +// } +// +// @Override +// public Result configIncrementCheckEnvironment(IncrementCheckConifg conifg) { +// return null; +// } +// }; +// +// } +//} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractSinkFeignClient.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractSinkFeignClient.java new file mode 100644 index 0000000..0ebc296 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractSinkFeignClient.java @@ -0,0 +1,17 @@ +package org.opengauss.datachecker.check.client; + +import org.springframework.cloud.openfeign.FeignClient; + +/** + * 创建一个内部类，声明被调用方的api接口，假如被调用方接口异常就会回调异常类进行异常声明 + * name可以声明value,datachecker-extract-sink 是服务名称直接调用该系统，名称一般采用eureka注册信息，我们未引入eurka,配置url进行调用 + * ExtractSinkFallBack 这是如果fegin调用失败需要熔断以及提示错误信息的类 + * + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +@FeignClient(name = "datachecker-extract-sink", url = "${data.check.sink-uri}") +public interface ExtractSinkFeignClient extends ExtractFeignClient { + +} \ No newline at end of file diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractSourceFeignClient.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractSourceFeignClient.java new file mode 100644 index 0000000..7a5030f --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/ExtractSourceFeignClient.java @@ -0,0 +1,17 @@ +package org.opengauss.datachecker.check.client; + +import org.springframework.cloud.openfeign.FeignClient; + +/** + * 创建一个内部类，声明被调用方的api接口，假如被调用方接口异常就会回调异常类进行异常声明 + * name可以声明value,datachecker-extract-source是服务名称直接调用该系统，名称一般采用eureka注册信息，我们未引入eurka,配置url进行调用 + * ExtractSourceFallBack 这是如果fegin调用失败需要熔断以及提示错误信息的类 + * + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +@FeignClient(name = "datachecker-extract-source", url = "${data.check.source-uri}") +public interface ExtractSourceFeignClient extends ExtractFeignClient { + +} \ No newline at end of file diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/FeignClientService.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/FeignClientService.java new file mode 100644 index 0000000..0e7f936 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/client/FeignClientService.java @@ -0,0 +1,189 @@ +package org.opengauss.datachecker.check.client; + +import org.opengauss.datachecker.common.entry.check.IncrementCheckConifg; +import org.opengauss.datachecker.common.entry.enums.DML; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.ExtractTask; +import org.opengauss.datachecker.common.entry.extract.SourceDataLog; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.opengauss.datachecker.common.entry.extract.Topic; +import org.opengauss.datachecker.common.exception.CheckingException; +import org.opengauss.datachecker.common.exception.DispatchClientException; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 实现FeignClient 接口调用封装 + * + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +@Service +public class FeignClientService { + @Autowired + private ExtractSourceFeignClient extractSourceClient; + + @Autowired + private ExtractSinkFeignClient extractSinkClient; + + /** + * 根据端点类型获取指定FeignClient + * + * @param endpoint 端点类型 + * @return feignClient + */ + public ExtractFeignClient getClient(@NonNull Endpoint endpoint) { + return Endpoint.SOURCE == endpoint ? extractSourceClient : extractSinkClient; + } + + /** + * 根据端点类型 获取对应健康状态 + * + * @param endpoint 端点类型 + * @return 健康状态 + */ + public Result health(@NonNull Endpoint endpoint) { + return getClient(endpoint).health(); + } + + /** + * 加载指定端点的数据库元数据信息 + * + * @param endpoint 端点类型 + * @return 元数据结果 + */ + public Map queryMetaDataOfSchema(@NonNull Endpoint endpoint) { + Result> result = getClient(endpoint).queryMetaDataOfSchema(); + if (result.isSuccess()) { + Map metadata = result.getData(); + return metadata; + } else { + //调度源端服务获取数据库元数据信息异常 + throw new DispatchClientException(endpoint, "The scheduling source service gets the database metadata information abnormally," + result.getMessage()); + } + } + + + /** + * 抽取任务构建 + * + * @param endpoint 端点类型 + * @param processNo 执行进程编号 + */ + public List buildExtractTaskAllTables(@NonNull Endpoint endpoint, String processNo) { + Result> result = getClient(endpoint).buildExtractTaskAllTables(processNo); + if (result.isSuccess()) { + return result.getData(); + } else { + //调度抽取服务构建任务异常 + throw new DispatchClientException(endpoint, "The scheduling extraction service construction task is abnormal," + result.getMessage()); + } + } + + public boolean buildExtractTaskAllTables(@NonNull Endpoint endpoint, String processNo, @NonNull List taskList) { + Result result = getClient(endpoint).buildExtractTaskAllTables(processNo, taskList); + if (result.isSuccess()) { + return result.isSuccess(); + } else { + //调度抽取服务构建任务异常 + throw new DispatchClientException(endpoint, "The scheduling extraction service construction task is abnormal," + result.getMessage()); + } + } + + /** + * 全量抽取业务处理流程 + * + * @param endpoint 端点类型 + * @param processNo 执行进程序号 + * @return 执行结果 + */ + public boolean execExtractTaskAllTables(@NonNull Endpoint endpoint, String processNo) { + Result result = getClient(endpoint).execExtractTaskAllTables(processNo); + if (result.isSuccess()) { + return result.isSuccess(); + } else { + //调度抽取服务执行任务失败 + throw new DispatchClientException(endpoint, "Scheduling extraction service execution task failed," + result.getMessage()); + } + } + + /** + * 清理对应端点构建的任务缓存信息 ，任务重置 + * + * @param endpoint 端点类型 + */ + public void cleanEnvironment(@NonNull Endpoint endpoint, String processNo) { + getClient(endpoint).cleanEnvironment(processNo); + } + + public void cleanTask(@NonNull Endpoint endpoint) { + getClient(endpoint).cleanTask(); + } + + /** + * 查询指定表对应的Topic信息 + * + * @param endpoint 端点类型 + * @param tableName 表名称 + * @return Topic信息 + */ + public Topic queryTopicInfo(@NonNull Endpoint endpoint, String tableName) { + Result result = getClient(endpoint).queryTopicInfo(tableName); + if (result.isSuccess()) { + return result.getData(); + } else { + return null; + } + } + + public Topic getIncrementTopicInfo(@NonNull Endpoint endpoint, String tableName) { + Result result = getClient(endpoint).getIncrementTopicInfo(tableName); + if (result.isSuccess()) { + return result.getData(); + } else { + return null; + } + } + + public List buildRepairDml(Endpoint endpoint, String schema, String tableName, DML dml, Set diffSet) { + Result> result = getClient(endpoint).buildRepairDml(schema, tableName, dml, diffSet); + if (result.isSuccess()) { + return result.getData(); + } else { + return null; + } + } + + /** + * 增量校验日志通知 + * + * @param endpoint 端点类型 + * @param dataLogList 增量校验日志 + */ + public void notifyIncrementDataLogs(Endpoint endpoint, List dataLogList) { + getClient(endpoint).notifyIncrementDataLogs(dataLogList); + } + + public String getDatabaseSchema(Endpoint endpoint) { + Result result = getClient(endpoint).getDatabaseSchema(); + if (result.isSuccess()) { + return result.getData(); + } else { + return null; + } + } + + public void configIncrementCheckEnvironment(Endpoint endpoint, IncrementCheckConifg conifg) { + Result result = getClient(endpoint).configIncrementCheckEnvironment(conifg); + if (!result.isSuccess()) { + throw new CheckingException(result.getMessage()); + } + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/AsyncConfig.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/AsyncConfig.java new file mode 100644 index 0000000..8643fac --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/AsyncConfig.java @@ -0,0 +1,38 @@ +package org.opengauss.datachecker.check.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * @author wang chao + * @date 2022/5/8 19:17 + * @since 11 + **/ +@Configuration +@EnableScheduling +public class AsyncConfig { + + @Bean("asyncCheckExecutor") + public ThreadPoolTaskExecutor asyncCheckExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + // 核心线程数， 当前机器的核心数 线程池创建时初始化线程数量 + executor.setCorePoolSize(Runtime.getRuntime().availableProcessors() * 2); + // 最大线程数：线程池最大的线程数，只有在缓冲队列满了之后才会申请超过核心线程数的线程 + executor.setMaxPoolSize(Runtime.getRuntime().availableProcessors() * 4); + // 缓冲队列： 用来缓冲执行任务的队列 + executor.setQueueCapacity(Integer.MAX_VALUE); + //允许线程空闲时间 + executor.setKeepAliveSeconds(60); + // 线程池名称前缀 + executor.setThreadNamePrefix("check-thread"); + // 缓冲队列满了之后的拒绝策略： + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + executor.initialize(); + return executor; + } + +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataCheckConfig.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataCheckConfig.java new file mode 100644 index 0000000..daa3e20 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataCheckConfig.java @@ -0,0 +1,46 @@ +package org.opengauss.datachecker.check.config; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.util.JsonObjectUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import javax.annotation.PostConstruct; +import java.io.File; + + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +@Slf4j +@Component +public class DataCheckConfig { + + private static final String CHECK_RESULT_PATH = File.separator + "Result" + File.separator + "Date" + File.separator; + + @Autowired + private DataCheckProperties properties; + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + + @PostConstruct + public DataCheckProperties getDataCheckProperties() { + log.info("check config properties [{}]", JsonObjectUtil.format(properties)); + return properties; + } + + public int getBucketCapacity() { + return properties.getBucketExpectCapacity(); + } + + public String getCheckResultPath() { + return properties.getDataPath() + CHECK_RESULT_PATH; + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataCheckProperties.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataCheckProperties.java new file mode 100644 index 0000000..e827635 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataCheckProperties.java @@ -0,0 +1,69 @@ +package org.opengauss.datachecker.check.config; + +import com.alibaba.fastjson.annotation.JSONType; +import lombok.Data; +import org.hibernate.validator.constraints.Range; +import org.opengauss.datachecker.common.entry.enums.CheckBlackWhiteMode; +import org.opengauss.datachecker.common.exception.CheckingAddressConflictException; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; +import org.springframework.validation.annotation.Validated; + +import javax.annotation.PostConstruct; +import javax.validation.constraints.NotEmpty; +import java.util.Objects; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +@Data +@Validated +@Component +@ConfigurationProperties(prefix = "data.check") +@JSONType(orders = {"sourceUri", "sinkUri", "bucketExpectCapacity", "healthCheckApi", "dataPath", "blackWhiteMode"}) +public class DataCheckProperties { + + @PostConstruct + private void checkUrl() { + if (Objects.equals(sourceUri, sinkUri)) { + // 源端和宿端的访问地址冲突，请重新配置。 + throw new CheckingAddressConflictException("The access addresses of the source end and the destination end conflict, please reconfigure."); + } + } + + + /** + * 数据校验服务地址：源端 源端地址不能为空 + */ + @NotEmpty(message = "Source address cannot be empty") + private String sourceUri; + + /** + * 数据校验服务地址：宿端 宿端地址不能为空") + */ + @NotEmpty(message = "The destination address cannot be empty") + private String sinkUri; + + /** + * 桶容量 默认容量大小为 1 + */ + @Range(min = 1, message = "The minimum barrel capacity is 1") + private int bucketExpectCapacity = 1; + + /** + * 健康检查地址 + */ + private String healthCheckApi; + /** + * 数据结果根目录,数据校验结果根目录不能为空 + */ + @NotEmpty(message = "The root directory of data verification results cannot be empty") + private String dataPath; + + /** + * 是否增加黑白名单配置 + */ + private CheckBlackWhiteMode blackWhiteMode; +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataSourceConfig.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataSourceConfig.java new file mode 100644 index 0000000..77605ad --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/DataSourceConfig.java @@ -0,0 +1,29 @@ +package org.opengauss.datachecker.check.config; + + +import com.alibaba.druid.pool.DruidDataSource; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.sql.DataSource; + + +@Configuration +public class DataSourceConfig { + /** + * 
+     *  将自定义的 Druid数据源添加到容器中，不再让 Spring Boot 自动创建
+     *  绑定全局配置文件中的 druid 数据源属性到 com.alibaba.druid.pool.DruidDataSource从而让它们生效
+     *  @ConfigurationProperties(prefix = "spring.datasource")：作用就是将 全局配置文件中
+     *  前缀为 spring.datasource的属性值注入到 com.alibaba.druid.pool.DruidDataSource 的同名参数中
+     *
+ * + * @return + */ + @Bean("dataSource") + @ConfigurationProperties(prefix = "spring.datasource.druid.datacheck") + public DataSource druidDataSourceOne() { + return new DruidDataSource(); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/GlobalCheckingExceptionHandler.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/GlobalCheckingExceptionHandler.java new file mode 100644 index 0000000..8b6453d --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/GlobalCheckingExceptionHandler.java @@ -0,0 +1,58 @@ +package org.opengauss.datachecker.check.config; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.entry.enums.ResultEnum; +import org.opengauss.datachecker.common.exception.*; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +import javax.servlet.http.HttpServletRequest; + +@Slf4j +@RestControllerAdvice +public class GlobalCheckingExceptionHandler extends GlobalCommonExceptionHandler { + + /** + * 业务异常处理 + */ + @ExceptionHandler(value = CheckingException.class) + public Result checkingException(HttpServletRequest request, CheckingException e) { + log.error("path:{}, queryParam:{}message:{}", request.getRequestURI(), request.getQueryString(), + e.getMessage(), e); + logError(request, e); + return Result.fail(ResultEnum.CHECKING, e.getMessage()); + } + + @ExceptionHandler(value = CheckingAddressConflictException.class) + public Result checkingAddressConflictException(HttpServletRequest request, CheckingAddressConflictException e) { + log.error("path:{}, queryParam:{}message:{}", request.getRequestURI(), request.getQueryString(), + e.getMessage(), e); + logError(request, e); + return Result.fail(ResultEnum.CHECKING_ADDRESS_CONFLICT, e.getMessage()); + } + + @ExceptionHandler(value = CheckMetaDataException.class) + public Result checkMetaDataException(HttpServletRequest request, CheckMetaDataException e) { + log.error("path:{}, queryParam:{}message:{}", request.getRequestURI(), request.getQueryString(), + e.getMessage(), e); + logError(request, e); + return Result.fail(ResultEnum.CHECK_META_DATA, e.getMessage()); + } + + @ExceptionHandler(value = LargeDataDiffException.class) + public Result largeDataDiffException(HttpServletRequest request, LargeDataDiffException e) { + log.error("path:{}, queryParam:{}message:{}", request.getRequestURI(), request.getQueryString(), + e.getMessage(), e); + logError(request, e); + return Result.fail(ResultEnum.LARGE_DATA_DIFF, e.getMessage()); + } + + @ExceptionHandler(value = MerkleTreeDepthException.class) + public Result merkleTreeDepthException(HttpServletRequest request, MerkleTreeDepthException e) { + log.error("path:{}, queryParam:{}message:{}", request.getRequestURI(), request.getQueryString(), + e.getMessage(), e); + logError(request, e); + return Result.fail(ResultEnum.MERKLE_TREE_DEPTH, e.getMessage()); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/SpringDocConfig.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/SpringDocConfig.java new file mode 100644 index 0000000..944edf7 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/config/SpringDocConfig.java @@ -0,0 +1,61 @@ +package org.opengauss.datachecker.check.config; + +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.parameters.HeaderParameter; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.springdoc.core.customizers.OpenApiCustomiser; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.util.ReflectionUtils; +import org.springframework.web.servlet.config.annotation.InterceptorRegistration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import java.lang.reflect.Field; +import java.util.List; + +/** + * swagger2配置 + * http://localhost:8080/swagger-ui/index.html + * + * @author ：wangchao + * @date ：Created in 2022/5/17 + * @since ：11 + */ + +/** + * 2021/8/13 + */ + +@Configuration +public class SpringDocConfig implements WebMvcConfigurer { + @Bean + public OpenAPI mallTinyOpenAPI() { + return new OpenAPI() + .info(new Info() + .title("data verification") + .description("Data verification tool data verification API") + .version("v1.0.0")); + } + + + /** + * 通用拦截器排除设置，所有拦截器都会自动加springdoc-opapi相关的资源排除信息，不用在应用程序自身拦截器定义的地方去添加，算是良心解耦实现。 + */ + @SuppressWarnings("unchecked") + @Override + public void addInterceptors(InterceptorRegistry registry) { + try { + Field registrationsField = FieldUtils.getField(InterceptorRegistry.class, "registrations", true); + List registrations = (List) ReflectionUtils.getField(registrationsField, registry); + if (registrations != null) { + for (InterceptorRegistration interceptorRegistration : registrations) { + interceptorRegistration.excludePathPatterns("/springdoc**/**"); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/CheckBlackWhiteController.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/CheckBlackWhiteController.java new file mode 100644 index 0000000..4895b0b --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/CheckBlackWhiteController.java @@ -0,0 +1,92 @@ +package org.opengauss.datachecker.check.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.opengauss.datachecker.check.service.CheckBlackWhiteService; +import org.opengauss.datachecker.check.service.CheckService; +import org.opengauss.datachecker.common.entry.enums.CheckMode; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/25 + * @since ：11 + */ +@Tag(name = "CheckBlackWhiteController", description = "校验服务-黑白名单管理") +@Validated +@RestController +@RequestMapping +public class CheckBlackWhiteController { + + @Autowired + private CheckBlackWhiteService checkBlackWhiteService; + + /** + * 开启校验 + */ + @Operation(summary = "添加白名单列表 该功能清理历史白名单，重置白名单为当前列表") + @PostMapping("/add/white/list") + public Result addWhiteList(@Parameter(name = "whiteList", description = "白名单列表") + @RequestBody List whiteList) { + checkBlackWhiteService.addWhiteList(whiteList); + return Result.success(); + } + + @Operation(summary = "更新白名单列表 该功能在当前白名单基础上新增当前列表到白名单") + @PostMapping("/update/white/list") + public Result updateWhiteList(@Parameter(name = "whiteList", description = "白名单列表") + @RequestBody List whiteList) { + checkBlackWhiteService.updateWhiteList(whiteList); + return Result.success(); + } + + @Operation(summary = "移除白名单列表 该功能在当前白名单基础上移除当前列表到白名单") + @PostMapping("/delete/white/list") + public Result deleteWhiteList(@Parameter(name = "whiteList", description = "白名单列表") + @RequestBody List whiteList) { + checkBlackWhiteService.deleteWhiteList(whiteList); + return Result.success(); + } + + @Operation(summary = "查询白名单列表 ") + @PostMapping("/query/white/list") + public Result> queryWhiteList() { + return Result.success(checkBlackWhiteService.queryWhiteList()); + } + + @Operation(summary = "添加黑名单列表 该功能清理历史黑名单，重置黑名单为当前列表") + @PostMapping("/add/black/list") + public Result addBlackList(@Parameter(name = "blackList", description = "黑名单列表") + @RequestBody List blackList) { + checkBlackWhiteService.addBlackList(blackList); + return Result.success(); + } + + @Operation(summary = "更新黑名单列表 该功能在当前黑名单基础上新增当前列表到黑名单") + @PostMapping("/update/black/list") + public Result updateBlackList(@Parameter(name = "blackList", description = "黑名单列表") + @RequestBody List blackList) { + checkBlackWhiteService.updateBlackList(blackList); + return Result.success(); + } + + @Operation(summary = "移除黑名单列表 该功能在当前黑名单基础上移除当前列表到黑名单") + @PostMapping("/delete/black/list") + public Result deleteBlackList(@Parameter(name = "blackList", description = "黑名单列表") + @RequestBody List blackList) { + checkBlackWhiteService.deleteBlackList(blackList); + return Result.success(); + } + + @Operation(summary = "查询黑名单列表 ") + @PostMapping("/query/black/list") + public Result> queryBlackList() { + return Result.success(checkBlackWhiteService.queryBlackList()); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/CheckStartController.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/CheckStartController.java new file mode 100644 index 0000000..d9d11ef --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/CheckStartController.java @@ -0,0 +1,57 @@ +package org.opengauss.datachecker.check.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.opengauss.datachecker.check.service.CheckService; +import org.opengauss.datachecker.common.entry.check.IncrementCheckConifg; +import org.opengauss.datachecker.common.entry.enums.CheckMode; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/25 + * @since ：11 + */ +@Tag(name = "CheckStartController", description = "校验服务-校验服务启动命令") +@Validated +@RestController +@RequestMapping +public class CheckStartController { + + @Autowired + private CheckService checkService; + + /** + * 开启校验 + */ + @Operation(summary = "开启校验") + @PostMapping("/start/check") + public Result statCheck(@Parameter(name = "checkMode", description = CheckMode.API_DESCRIPTION) + @RequestParam("checkMode") CheckMode checkMode) { + return Result.success(checkService.start(checkMode)); + } + + @Operation(summary = "增量校验配置初始化") + @PostMapping("/increment/check/config") + public Result incrementCheckConifg(@RequestBody IncrementCheckConifg incrementCheckConifg) { + checkService.incrementCheckConifg(incrementCheckConifg); + return Result.success(); + } + + @Operation(summary = "停止校验服务 并 清理校验服务", description = "对当前进程中的校验状态，以及抽取的数据等相关信息进行全面清理。") + @PostMapping("/stop/clean/check") + public Result cleanCheck() { + checkService.cleanCheck(); + return Result.success(); + } + + @Operation(summary = "查询当前校验服务进程编号") + @GetMapping("/get/check/process") + public Result getCurrentCheckProcess() { + return Result.success(checkService.getCurrentCheckProcess()); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/IncrementManagerController.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/IncrementManagerController.java new file mode 100644 index 0000000..9d5a96a --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/IncrementManagerController.java @@ -0,0 +1,43 @@ +package org.opengauss.datachecker.check.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.opengauss.datachecker.check.service.IncrementManagerService; +import org.opengauss.datachecker.common.entry.extract.SourceDataLog; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.constraints.NotEmpty; +import java.util.List; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/25 + * @since ：11 + */ +@Tag(name = "IncrementManagerController", description = "校验服务-增量校验管理") +@Validated +@RestController +public class IncrementManagerController { + + @Autowired + private IncrementManagerService incrementManagerService; + + /** + * 增量校验日志通知 + * + * @param dataLogList 增量校验日志 + */ + @Operation(summary = "增量校验日志通知") + @PostMapping("/notify/source/increment/data/logs") + public void notifySourceIncrementDataLogs(@Parameter(description = "增量校验日志") + @RequestBody @NotEmpty List dataLogList) { + incrementManagerService.notifySourceIncrementDataLogs(dataLogList); + } + + +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/TaskStatusController.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/TaskStatusController.java new file mode 100644 index 0000000..33a1144 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/controller/TaskStatusController.java @@ -0,0 +1,57 @@ +package org.opengauss.datachecker.check.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.opengauss.datachecker.check.modules.task.TaskManagerService; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.lang.NonNull; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.constraints.NotEmpty; +import java.util.List; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/25 + * @since ：11 + */ +@Tag(name = "TaskStatusController", description = "校验服务-数据抽取任务状态") +@Validated +@RestController +public class TaskStatusController { + + @Autowired + private TaskManagerService taskManagerService; + + /** + * 刷新指定任务的数据抽取表执行状态 + * + * @param tableName 表名称 + * @param endpoint 端点类型 {@link org.opengauss.datachecker.common.entry.enums.Endpoint} + */ + @Operation(summary = "刷新指定任务的数据抽取任务执行状态") + @PostMapping("/table/extract/status") + public void refushTableExtractStatus(@Parameter(description = "表名称") @RequestParam(value = "tableName") @NotEmpty String tableName, + @Parameter(description = "数据校验端点类型") @RequestParam(value = "endpoint") @NonNull Endpoint endpoint) { + taskManagerService.refushTableExtractStatus(tableName, endpoint); + } + + /** + * 初始化任务状态 + * + * @param tableNameList 表名称列表 + */ + @Operation(summary = "初始化任务状态") + @PostMapping("/table/extract/status/init") + public void initTableExtractStatus(@Parameter(description = "表名称列表") @RequestBody @NotEmpty List tableNameList) { + taskManagerService.initTableExtractStatus(tableNameList); + } + + +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/bucket/Bucket.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/bucket/Bucket.java new file mode 100644 index 0000000..6897555 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/bucket/Bucket.java @@ -0,0 +1,73 @@ +package org.opengauss.datachecker.check.modules.bucket; + +import lombok.Data; +import lombok.experimental.Accessors; +import org.opengauss.datachecker.common.entry.extract.RowDataHash; +import org.opengauss.datachecker.common.util.ByteUtil; + +import javax.validation.constraints.NotNull; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +@Data +@Accessors(chain = true) +public class Bucket implements Serializable { + + private static final long serialVersionUID = 1L; + /** + * 桶初始化容量，如果桶内数据的数量超过指定数量{@code initialCapacity*0.75} 则会自动触发桶容量扩容。 + */ + private int initialCapacity; + + /** + * Bucket桶的容器,容器的初始化容量大小为设置为平均容量大小。 + *

+ * 超出平均容量会进行扩容操作 + */ + private Map bucket = new HashMap<>(this.initialCapacity); + /** + * 桶编号 + */ + private Integer number; + /** + * Bucket桶的哈希签名 ,签名初始化值为0 + */ + private long signature = 0L; + + /** + * 桶构造时，要求进行容量大小初始化 + * + * @param initialCapacity 桶初始化容量大小 + */ + public Bucket(int initialCapacity) { + this.initialCapacity = initialCapacity; + } + + /** + * 将行记录哈希对象添加到桶容器中。并计算桶的哈希签名。 + *

+ * 桶的哈希签名算法为当前桶的哈希签名{@code signature}异或当前插入记录的行哈希值。 + * + * @param rowDataHash 行记录哈希对象 + * @return 返回插入集合结果 + */ + public RowDataHash put(@NotNull RowDataHash rowDataHash) { + signature = signature ^ rowDataHash.getRowHash(); + return bucket.put(rowDataHash.getPrimaryKey(), rowDataHash); + } + + /** + * 获取当前桶的哈希签名 + * + * @return 桶的哈希签名 + */ + public byte[] getSignature() { + return ByteUtil.toBytes(signature); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/bucket/BuilderBucketHandler.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/bucket/BuilderBucketHandler.java new file mode 100644 index 0000000..c6bca8f --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/bucket/BuilderBucketHandler.java @@ -0,0 +1,124 @@ +package org.opengauss.datachecker.check.modules.bucket; + +import org.opengauss.datachecker.common.entry.extract.RowDataHash; +import org.springframework.lang.NonNull; + +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/24 + * @since ：11 + */ +public class BuilderBucketHandler { + /** + * 默克尔树最大树高度 + */ + private static final int MERKLE_TREE_MAX_HEIGHT = 15; + /** + * 最高默克尔树的最大叶子节点数量 + */ + private static final int BUCKET_MAX_COUNT_LIMITS = 1 << MERKLE_TREE_MAX_HEIGHT; + + + /** + * 当限定了默克尔树最大树高度为{@value MERKLE_TREE_MAX_HEIGHT}， + * 那么构造的最高默克尔树的最大叶子节点数量为{@code BUCKET_MAX_COUNT_LIMITS} 即 {@value BUCKET_MAX_COUNT_LIMITS}。 + *

+ * 由此，获得最大桶数量值为{@value BUCKET_MAX_COUNT_LIMITS }， + * 桶数量范围我们限定每棵树桶的数量为 2^n 个 + */ + private static final int[] BUCKET_COUNT_LIMITS = new int[MERKLE_TREE_MAX_HEIGHT]; + + // 初始化{@code BUCKET_COUNT_LIMITS} + static { + for (int i = 1; i <= MERKLE_TREE_MAX_HEIGHT; i++) { + BUCKET_COUNT_LIMITS[i - 1] = 1 << i; + } + } + + /** + * 空桶容量大小，用于构造特殊的空桶 + */ + private static final int EMPTY_INITIAL_CAPACITY = 0; + + /** + * 当前桶初始化容量 + */ + private final int bucketCapacity; + + public BuilderBucketHandler(int bucketCapacity) { + this.bucketCapacity = bucketCapacity; + } + + /** + * 将{@code rowDataHashList}数据动态分配到桶{@link org.opengauss.datachecker.check.modules.bucket.Bucket}中。 + *

+ * + * @param rowDataHashList 当前待分配到桶中的记录集合 + * @param totalCount 为{@link org.opengauss.datachecker.common.entry.extract.RowDataHash} 记录总数。 + * 注意：不一定为当前{@code rowDataHashList.size}总数 + * @param bucketMap {@code bucketMap} K为当前桶V的编号。 + */ + public void builder(@NonNull List rowDataHashList, int totalCount, @NonNull Map bucketMap) { + // 根据当前记录总数计算当前最大桶数量 + int maxBucketCount = calacMaxBucketCount(totalCount); + // 桶平均容量-用于初始化桶容量大小 + int averageCapacity = totalCount / maxBucketCount; + rowDataHashList.forEach(row -> { + long primaryKeyHash = row.getPrimaryKeyHash(); + // 计算桶编号信息 + int bucketNumber = calacBucketNumber(primaryKeyHash, maxBucketCount); + Bucket bucket; + // 根据row 信息获取指定编号的桶，如果不存在则创建桶 + if (bucketMap.containsKey(bucketNumber)) { + bucket = bucketMap.get(bucketNumber); + } else { + bucket = new Bucket(averageCapacity).setNumber(bucketNumber); + bucketMap.put(bucketNumber, bucket); + } + // 将row 添加到指定桶编号的桶中 + bucket.put(row); + }); + + } + + /** + * 根据{@code totalCount}记录总数计算当前最大桶数量。桶的数量为2^n个 + * + * @param totalCount 记录总数 + * @return 最大桶数量 + */ + private int calacMaxBucketCount(int totalCount) { + int bucketCount = totalCount / bucketCapacity; + int asInt = IntStream.range(0, 15) + .filter(idx -> BUCKET_COUNT_LIMITS[idx] > bucketCount) + .findFirst() + .orElse(15); + return BUCKET_COUNT_LIMITS[asInt]; + } + + /** + * 根据{@code rowHash}值对当前记录进行标记，此标记用于桶的编号 + * + * @param primaryKeyHash 行记录主键哈希值 + * @param bucketCount 桶数量 桶的数量为2^n个 + * @return 行记录桶编号 + */ + private int calacBucketNumber(long primaryKeyHash, int bucketCount) { + return (int) (Math.abs(primaryKeyHash) & (bucketCount - 1)); + } + + + /** + * 根据编号构造空桶 + * + * @param bucketNumber 桶编号 + * @return 桶 + */ + public static Bucket builderEmpty(Integer bucketNumber) { + return new Bucket(EMPTY_INITIAL_CAPACITY).setNumber(bucketNumber); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/AbstractCheckDiffResultBuilder.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/AbstractCheckDiffResultBuilder.java new file mode 100644 index 0000000..02f45f7 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/AbstractCheckDiffResultBuilder.java @@ -0,0 +1,125 @@ +package org.opengauss.datachecker.check.modules.check; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.check.client.FeignClientService; +import org.opengauss.datachecker.common.entry.enums.DML; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.exception.DispatchClientException; +import org.springframework.util.CollectionUtils; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/18 + * @since ：11 + */ +@Slf4j +@Getter +public abstract class AbstractCheckDiffResultBuilder> { + + private final FeignClientService feignClient; + private String table; + private int partitions; + private String topic; + private String schema; + private LocalDateTime createTime; + private Set keyUpdateSet; + private Set keyInsertSet; + private Set keyDeleteSet; + + private List repairUpdate; + private List repairInsert; + private List repairDelete; + + public AbstractCheckDiffResultBuilder(FeignClientService feignClient) { + this.feignClient = feignClient; + } + + /** + * @return 返回构建器自身对象 + */ + protected abstract B self(); + + /** + * @return 执行构建器 + */ + public abstract C build(); + + public B table(String table) { + this.table = table; + return this.self(); + } + + public B topic(String topic) { + this.topic = topic; + return this.self(); + } + + public B schema(String schema) { + this.schema = schema; + return this.self(); + } + + public B partitions(int partitions) { + this.partitions = partitions; + return this.self(); + } + + public B keyUpdateSet(Set keyUpdateSet) { + this.keyUpdateSet = keyUpdateSet; + this.repairUpdate = checkRepairSinkDiff(DML.REPLACE, this.schema, this.table, this.keyUpdateSet); + return this.self(); + } + + public B keyInsertSet(Set keyInsertSet) { + this.keyInsertSet = keyInsertSet; + this.repairInsert = checkRepairSinkDiff(DML.INSERT, this.schema, this.table, this.keyInsertSet); + return this.self(); + } + + public B keyDeleteSet(Set keyDeleteSet) { + this.keyDeleteSet = keyDeleteSet; + this.repairDelete = checkRepairSinkDiff(DML.DELETE, this.schema, this.table, this.keyDeleteSet); + return this.self(); + } + + + public static AbstractCheckDiffResultBuilder builder(FeignClientService feignClient) { + return new AbstractCheckDiffResultBuilderImpl(feignClient); + } + + private static final class AbstractCheckDiffResultBuilderImpl extends AbstractCheckDiffResultBuilder { + private AbstractCheckDiffResultBuilderImpl(FeignClientService feignClient) { + super(feignClient); + } + + @Override + protected AbstractCheckDiffResultBuilderImpl self() { + return this; + } + + @Override + public CheckDiffResult build() { + super.createTime = LocalDateTime.now(); + return new CheckDiffResult(this); + } + } + + private List checkRepairSinkDiff(DML dml, String schema, String tableName, Set sinkDiffSet) { + if (!CollectionUtils.isEmpty(sinkDiffSet)) { + try { + return feignClient.buildRepairDml(Endpoint.SOURCE, schema, tableName, dml, sinkDiffSet); + } catch (DispatchClientException exception) { + log.error("check table[{}] Repair [{}] Diff build Repair DML Error", tableName, dml, exception); + return new ArrayList<>(); + } + } else { + return new ArrayList<>(); + } + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckDiffResult.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckDiffResult.java new file mode 100644 index 0000000..5543652 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/CheckDiffResult.java @@ -0,0 +1,47 @@ +package org.opengauss.datachecker.check.modules.check; + +import com.alibaba.fastjson.annotation.JSONType; +import lombok.Data; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Set; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/18 + * @since ：11 + */ +@Data +@JSONType(orders = {"schema", "table", "topic", "partitions", "createTime", + "keyInsertSet", "keyUpdateSet", "keyDeleteSet", + "repairInsert", "repairUpdate", "repairDelete"}) +public class CheckDiffResult { + private String schema; + private String table; + private String topic; + private int partitions; + private LocalDateTime createTime; + + private Set keyInsertSet; + private Set keyUpdateSet; + private Set keyDeleteSet; + + private List repairInsert; + private List repairUpdate; + private List repairDelete; + + public CheckDiffResult(final AbstractCheckDiffResultBuilder builder) { + this.table = builder.getTable(); + this.partitions = builder.getPartitions(); + this.topic = builder.getTopic(); + this.schema = builder.getSchema(); + this.createTime = builder.getCreateTime(); + this.keyUpdateSet = builder.getKeyUpdateSet(); + this.keyInsertSet = builder.getKeyInsertSet(); + this.keyDeleteSet = builder.getKeyDeleteSet(); + this.repairUpdate = builder.getRepairUpdate(); + this.repairInsert = builder.getRepairInsert(); + this.repairDelete = builder.getRepairDelete(); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckService.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckService.java new file mode 100644 index 0000000..ed336db --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckService.java @@ -0,0 +1,54 @@ +package org.opengauss.datachecker.check.modules.check; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.check.client.FeignClientService; +import org.opengauss.datachecker.check.config.DataCheckConfig; +import org.opengauss.datachecker.common.entry.check.DataCheckParam; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.Topic; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.lang.NonNull; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +@Slf4j +@Service +public class DataCheckService { + + @Autowired + private FeignClientService feignClientService; + + @Autowired + private DataCheckConfig dataCheckConfig; + + @Autowired + @Qualifier("asyncCheckExecutor") + private ThreadPoolTaskExecutor checkAsyncExecutor; + + /** + * @param topic + * @param partitions + */ + public void checkTableData(@NonNull Topic topic, int partitions) { + final int bucketCapacity = dataCheckConfig.getBucketCapacity(); + final String checkResultPath = dataCheckConfig.getCheckResultPath(); + + String schema = feignClientService.getDatabaseSchema(Endpoint.SINK); + final DataCheckParam checkParam = new DataCheckParam(bucketCapacity, topic, partitions, checkResultPath, schema); + checkAsyncExecutor.submit(new DataCheckThread(checkParam, feignClientService)); + } + + public void incrementCheckTableData(Topic topic) { + final int bucketCapacity = dataCheckConfig.getBucketCapacity(); + final String checkResultPath = dataCheckConfig.getCheckResultPath(); + String schema = feignClientService.getDatabaseSchema(Endpoint.SINK); + final DataCheckParam checkParam = new DataCheckParam(bucketCapacity, topic, 0, checkResultPath, schema); + checkAsyncExecutor.submit(new IncrementDataCheckThread(checkParam, feignClientService)); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckThread.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckThread.java new file mode 100644 index 0000000..6943d63 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckThread.java @@ -0,0 +1,341 @@ +package org.opengauss.datachecker.check.modules.check; + +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.check.client.FeignClientService; +import org.opengauss.datachecker.check.modules.bucket.Bucket; +import org.opengauss.datachecker.check.modules.bucket.BuilderBucketHandler; +import org.opengauss.datachecker.check.modules.merkle.MerkleTree; +import org.opengauss.datachecker.check.modules.merkle.MerkleTree.Node; +import org.opengauss.datachecker.common.constant.Constants; +import org.opengauss.datachecker.common.entry.check.DataCheckParam; +import org.opengauss.datachecker.common.entry.check.DifferencePair; +import org.opengauss.datachecker.common.entry.check.Pair; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.RowDataHash; +import org.opengauss.datachecker.common.entry.extract.Topic; +import org.opengauss.datachecker.common.exception.LargeDataDiffException; +import org.opengauss.datachecker.common.exception.MerkleTreeDepthException; +import org.springframework.lang.NonNull; +import org.springframework.util.CollectionUtils; + +import java.util.*; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +@Slf4j +public class DataCheckThread implements Runnable { + private static final int THRESHOLD_MIN_BUCKET_SIZE = 2; + private static final String THREAD_NAME_PRIFEX = "data-check-"; + + private final Topic topic; + private final String tableName; + private final int partitions; + private final int bucketCapacity; + private final String path; + private final String sinkSchema; + + private final FeignClientService feignClient; + + private final List sourceBucketList = new ArrayList<>(); + private final List sinkBucketList = new ArrayList<>(); + private final DifferencePair, Map, Map>> difference + = DifferencePair.of(new HashMap<>(), new HashMap<>(), new HashMap<>()); + + private final Map> bucketNumberDiffMap = new HashMap<>(); + private final QueryRowDataWapper queryRowDataWapper; + private final DataCheckWapper dataCheckWapper; + + public DataCheckThread(@NonNull DataCheckParam checkParam, @NonNull FeignClientService feignClient) { + this.topic = checkParam.getTopic(); + this.tableName = topic.getTableName(); + this.partitions = checkParam.getPartitions(); + this.path = checkParam.getPath(); + this.sinkSchema = checkParam.getSchema(); + this.bucketCapacity = checkParam.getBucketCapacity(); + this.feignClient = feignClient; + this.queryRowDataWapper = new QueryRowDataWapper(feignClient); + this.dataCheckWapper = new DataCheckWapper(); + resetThreadName(); + } + + /** + * When an object implementing interface Runnable is used + * to create a thread, starting the thread causes the object's + * run method to be called in that separately executing + * thread. + *

+ * The general contract of the method run is that it may + * take any action whatsoever. + * + * @see Thread#run() + */ + @Override + public void run() { + + // 初始化桶列表 + initBucketList(); + + //不进行默克尔树校验算法场景 + if (checkNotMerkleCompare(sourceBucketList.size(), sinkBucketList.size())) { + return; + } + + // 构造默克尔树 约束: bucketList 不能为空，且size>=2 + MerkleTree sourceTree = new MerkleTree(sourceBucketList); + MerkleTree sinkTree = new MerkleTree(sinkBucketList); + + // 默克尔树比较 + if (sourceTree.getDepth() != sinkTree.getDepth()) { + throw new MerkleTreeDepthException(String.format("source & sink data have large different, Please synchronize data again! " + + "merkel tree depth different,source depth=[%d],sink depth=[%d]", + sourceTree.getDepth(), sinkTree.getDepth())); + } + //递归比较两颗默克尔树，并将差异记录返回。 + compareMerkleTree(sourceTree, sinkTree); + // 校验结果 校验修复报告 + checkResult(); + cleanCheckThreadEnvironment(); + } + + private void cleanCheckThreadEnvironment() { + bucketNumberDiffMap.clear(); + sourceBucketList.clear(); + sinkBucketList.clear(); + difference.getOnlyOnLeft().clear(); + difference.getOnlyOnRight().clear(); + difference.getDiffering().clear(); + } + + + /** + * 初始化桶列表 + */ + private void initBucketList() { + // 获取当前任务对应kafka分区号 + // 初始化源端桶列列表数据 + initBucketList(Endpoint.SOURCE, partitions, sourceBucketList); + // 初始化宿端桶列列表数据 + initBucketList(Endpoint.SINK, partitions, sinkBucketList); + // 对齐源端宿端桶列表 + alignAllBuckets(); + // 排序 + sortBuckets(sourceBucketList); + sortBuckets(sinkBucketList); + } + + /** + * 根据桶编号对最终桶列表进行排序 + * + * @param bucketList 桶列表 + */ + private void sortBuckets(@NonNull List bucketList) { + bucketList.sort(Comparator.comparingInt(Bucket::getNumber)); + } + + /** + * 根据统计的源端宿端桶差异信息{@code bucketNumberDiffMap}结果，对齐桶列表数据。 + */ + private void alignAllBuckets() { + dataCheckWapper.alignAllBuckets(bucketNumberDiffMap, sourceBucketList, sinkBucketList); + } + + /** + * 拉取指定端点{@code endpoint}服务当前表{@code tableName}的kafka分区{@code partitions}数据。 + * 并将kafka数据分组组装到指定的桶列表{@code bucketList}中 + * + * @param endpoint 端点类型 + * @param partitions kafka分区号 + * @param bucketList 桶列表 + */ + private void initBucketList(Endpoint endpoint, int partitions, List bucketList) { + Map bucketMap = new HashMap<>(Constants.InitialCapacity.MAP); + // 使用feignclient 拉取kafka数据 + List dataList = getTopicPartitionsData(endpoint, partitions); + if (CollectionUtils.isEmpty(dataList)) { + return; + } + BuilderBucketHandler bucketBuilder = new BuilderBucketHandler(bucketCapacity); + + // 拉取的数据进行构建桶列表 + bucketBuilder.builder(dataList, dataList.size(), bucketMap); + // 统计桶列表信息 + bucketNoStatistics(endpoint, bucketMap.keySet()); + bucketList.addAll(bucketMap.values()); + } + + /** + * 比较两颗默克尔树，并将差异记录返回。 + * + * @param sourceTree 源端默克尔树 + * @param sinkTree 宿端默克尔树 + */ + private void compareMerkleTree(@NonNull MerkleTree sourceTree, @NonNull MerkleTree sinkTree) { + Node source = sourceTree.getRoot(); + Node sink = sinkTree.getRoot(); + List> diffNodeList = new LinkedList<>(); + compareMerkleTree(source, sink, diffNodeList); + if (CollectionUtils.isEmpty(diffNodeList)) { + return; + } + diffNodeList.forEach(diffNode -> { + Bucket sourceBucket = diffNode.getSource().getBucket(); + Bucket sinkBucket = diffNode.getSink().getBucket(); + DifferencePair subDifference = compareBucket(sourceBucket, sinkBucket); + difference.getDiffering().putAll(subDifference.getDiffering()); + difference.getOnlyOnLeft().putAll(subDifference.getOnlyOnLeft()); + difference.getOnlyOnRight().putAll(subDifference.getOnlyOnRight()); + }); + + } + + /** + * 比较两个桶内部记录的差异数据 + *

+ * 差异类型 {@linkplain org.opengauss.datachecker.common.entry.enums.DiffCategory} + * + * @param sourceBucket 源端桶 + * @param sinkBucket 宿端桶 + * @return 差异记录 + */ + private DifferencePair compareBucket(Bucket sourceBucket, Bucket sinkBucket) { + + Map sourceMap = sourceBucket.getBucket(); + Map sinkMap = sinkBucket.getBucket(); + + MapDifference difference = Maps.difference(sourceMap, sinkMap); + + Map entriesOnlyOnLeft = difference.entriesOnlyOnLeft(); + Map entriesOnlyOnRight = difference.entriesOnlyOnRight(); + Map> entriesDiffering = difference.entriesDiffering(); + Map> differing = new HashMap<>(Constants.InitialCapacity.MAP); + entriesDiffering.forEach((key, diff) -> { + differing.put(key, Pair.of(diff.leftValue(), diff.rightValue())); + }); + return DifferencePair.of(entriesOnlyOnLeft, entriesOnlyOnRight, differing); + } + + /** + * 递归比较两颗默克尔树节点，并记录差异节点。 + *

+ * 采用递归-前序遍历方式，遍历比较默克尔树，从而查找差异节点。 + *

+ * 若当前遍历的节点{@link org.opengauss.datachecker.check.modules.merkle.MerkleTree.Node}签名相同则终止当前遍历分支。 + * + * @param source 源端默克尔树节点 + * @param sink 宿端默克尔树节点 + * @param diffNodeList 差异节点记录 + */ + private void compareMerkleTree(@NonNull Node source, @NonNull Node sink, List> diffNodeList) { + // 如果节点相同，则退出 + if (Arrays.equals(source.getSignature(), sink.getSignature())) { + return; + } + // 如果节点不相同，则继续比较下层节点，若当前差异节点为叶子节点，则记录该差异节点，并退出 + if (source.getType() == MerkleTree.LEAF_SIG_TYPE) { + diffNodeList.add(Pair.of(source, sink)); + return; + } + compareMerkleTree(source.getLeft(), sink.getLeft(), diffNodeList); + + compareMerkleTree(source.getRight(), sink.getRight(), diffNodeList); + + } + + /** + * 对各端点构建的桶编号进行统计。统计结果汇总到{@code bucketNumberDiffMap}中。 + *

+ * 默克尔比较算法，需要确保双方桶编号的一致。 + *

+ * 如果一方的桶编号存在缺失，即{@code Pair}中，S或T的值为-1，则需要生成相应编号的空桶。 + * + * @param endpoint 端点 + * @param bucketNoSet 桶编号 + */ + private void bucketNoStatistics(@NonNull Endpoint endpoint, @NonNull Set bucketNoSet) { + bucketNoSet.forEach(bucketNo -> { + if (!bucketNumberDiffMap.containsKey(bucketNo)) { + if (endpoint == Endpoint.SOURCE) { + bucketNumberDiffMap.put(bucketNo, Pair.of(bucketNo, -1)); + } else { + bucketNumberDiffMap.put(bucketNo, Pair.of(-1, bucketNo)); + } + } else { + Pair pair = bucketNumberDiffMap.get(bucketNo); + if (endpoint == Endpoint.SOURCE) { + bucketNumberDiffMap.put(bucketNo, Pair.of(bucketNo, pair)); + } else { + bucketNumberDiffMap.put(bucketNo, Pair.of(pair, bucketNo)); + } + } + }); + } + + /** + * 拉取指定端点{@code endpoint}的表{@code tableName}的 kafka分区{@code partitions}数据 + * + * @param endpoint 端点类型 + * @param partitions kafka分区号 + * @return 指定表 kafka分区数据 + */ + private List getTopicPartitionsData(Endpoint endpoint, int partitions) { + return queryRowDataWapper.queryRowData(endpoint, tableName, partitions); + } + + /** + * 不满足默克尔树约束条件下 比较 + * + * @param sourceBucketCount 源端数量 + * @param sinkBucketCount 宿端数量 + * @return 是否满足默克尔校验场景 + */ + private boolean checkNotMerkleCompare(int sourceBucketCount, int sinkBucketCount) { + // 满足构造默克尔树约束条件 + if (sourceBucketCount >= THRESHOLD_MIN_BUCKET_SIZE && sinkBucketCount >= THRESHOLD_MIN_BUCKET_SIZE) { + return false; + } + // 不满足默克尔树约束条件下 比较 + if (sourceBucketCount == sinkBucketCount) { + // sourceSize等于0，即都是空桶 + if (sourceBucketCount == 0) { + //表是空表, 校验成功! + log.info("table[{}] is an empty table,this check successful!", tableName); + } else { + // sourceSize小于thresholdMinBucketSize 即都只有一个桶，比较 + DifferencePair subDifference = compareBucket(sourceBucketList.get(0), sinkBucketList.get(0)); + difference.getDiffering().putAll(subDifference.getDiffering()); + difference.getOnlyOnLeft().putAll(subDifference.getOnlyOnLeft()); + difference.getOnlyOnRight().putAll(subDifference.getOnlyOnRight()); + } + } else { + throw new LargeDataDiffException(String.format("table[%s] source & sink data have large different," + + "source-bucket-count=[%s] sink-bucket-count=[%s]" + + " Please synchronize data again! ", tableName, sourceBucketCount, sinkBucketCount)); + } + return true; + } + + private void checkResult() { + CheckDiffResult result = AbstractCheckDiffResultBuilder.builder(feignClient) + .table(tableName) + .topic(topic.getTopicName()) + .schema(sinkSchema) + .partitions(partitions) + .keyUpdateSet(difference.getDiffering().keySet()) + .keyInsertSet(difference.getOnlyOnLeft().keySet()) + .keyDeleteSet(difference.getOnlyOnRight().keySet()) + .build(); + ExportCheckResult.export(path, result); + } + + /** + * 重置当前线程 线程名称 + */ + private void resetThreadName() { + Thread.currentThread().setName(THREAD_NAME_PRIFEX + topic.getTopicName()); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckWapper.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckWapper.java new file mode 100644 index 0000000..c6bbe30 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/DataCheckWapper.java @@ -0,0 +1,40 @@ +package org.opengauss.datachecker.check.modules.check; + +import org.opengauss.datachecker.check.modules.bucket.Bucket; +import org.opengauss.datachecker.check.modules.bucket.BuilderBucketHandler; +import org.opengauss.datachecker.common.entry.check.Pair; +import org.springframework.lang.NonNull; +import org.springframework.util.CollectionUtils; + +import java.util.List; +import java.util.Map; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/18 + * @since ：11 + */ +public class DataCheckWapper { + + + /** + * 根据统计的源端宿端桶差异信息{@code bucketNumberDiffMap}结果，对齐桶列表数据。 + * + * @param bucketNumberDiffMap 源端宿端桶差异信息 + * @param sourceBucketList 源端桶列表 + * @param sinkBucketList 宿端通列表 + */ + public void alignAllBuckets(Map> bucketNumberDiffMap, + @NonNull List sourceBucketList, @NonNull List sinkBucketList) { + if (!CollectionUtils.isEmpty(bucketNumberDiffMap)) { + bucketNumberDiffMap.forEach((number, pair) -> { + if (pair.getSource() == -1) { + sourceBucketList.add(BuilderBucketHandler.builderEmpty(number)); + } + if (pair.getSink() == -1) { + sinkBucketList.add(BuilderBucketHandler.builderEmpty(number)); + } + }); + } + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/ExportCheckResult.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/ExportCheckResult.java new file mode 100644 index 0000000..e61361e --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/ExportCheckResult.java @@ -0,0 +1,22 @@ +package org.opengauss.datachecker.check.modules.check; + +import org.opengauss.datachecker.common.util.FileUtils; +import org.opengauss.datachecker.common.util.JsonObjectUtil; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/17 + * @since ：11 + */ +public class ExportCheckResult { + + public static void export(String path, CheckDiffResult result) { + FileUtils.createDirectories(path); + String fileName = getCheckResultFileName(path, result.getTable(), result.getPartitions()); + FileUtils.writeAppendFile(fileName, JsonObjectUtil.format(result)); + } + + private static String getCheckResultFileName(String path, String tableName, int partitions) { + return path.concat(tableName).concat("_").concat(String.valueOf(partitions)).concat(".txt"); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/IncrementDataCheckThread.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/IncrementDataCheckThread.java new file mode 100644 index 0000000..15581c5 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/IncrementDataCheckThread.java @@ -0,0 +1,417 @@ +package org.opengauss.datachecker.check.modules.check; + +import com.google.common.collect.MapDifference; +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.check.client.FeignClientService; +import org.opengauss.datachecker.check.modules.bucket.Bucket; +import org.opengauss.datachecker.check.modules.bucket.BuilderBucketHandler; +import org.opengauss.datachecker.check.modules.merkle.MerkleTree; +import org.opengauss.datachecker.check.modules.merkle.MerkleTree.Node; +import org.opengauss.datachecker.common.entry.check.DataCheckParam; +import org.opengauss.datachecker.common.entry.check.DifferencePair; +import org.opengauss.datachecker.common.entry.check.Pair; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.RowDataHash; +import org.opengauss.datachecker.common.entry.extract.SourceDataLog; +import org.opengauss.datachecker.common.entry.extract.TableMetadataHash; +import org.opengauss.datachecker.common.entry.extract.Topic; +import org.opengauss.datachecker.common.exception.DispatchClientException; +import org.opengauss.datachecker.common.exception.MerkleTreeDepthException; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.lang.NonNull; +import org.springframework.util.CollectionUtils; + +import java.util.*; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +@Slf4j +public class IncrementDataCheckThread implements Runnable { + private static final int THRESHOLD_MIN_BUCKET_SIZE = 2; + private static final String THREAD_NAME_PRIFEX = "increment-data-check-"; + + private final Topic topic; + private final String tableName; + private final int partitions; + private final int bucketCapacity; + private final String path; + + private final FeignClientService feignClient; + + private final List sourceBucketList = new ArrayList<>(); + private final List sinkBucketList = new ArrayList<>(); + private final DifferencePair, Map, Map>> difference + = DifferencePair.of(new HashMap<>(), new HashMap<>(), new HashMap<>()); + + private final Map> bucketNumberDiffMap = new HashMap<>(); + private final QueryRowDataWapper queryRowDataWapper; + private final DataCheckWapper dataCheckWapper; + + public IncrementDataCheckThread(@NonNull DataCheckParam checkParam, @NonNull FeignClientService feignClient) { + this.topic = checkParam.getTopic(); + this.tableName = topic.getTableName(); + this.partitions = checkParam.getPartitions(); + this.path = checkParam.getPath(); + this.bucketCapacity = checkParam.getBucketCapacity(); + this.feignClient = feignClient; + this.queryRowDataWapper = new QueryRowDataWapper(feignClient); + this.dataCheckWapper = new DataCheckWapper(); + resetThreadName(); + } + + /** + * When an object implementing interface Runnable is used + * to create a thread, starting the thread causes the object's + * run method to be called in that separately executing + * thread. + *

+ * The general contract of the method run is that it may + * take any action whatsoever. + * + * @see Thread#run() + */ + @Override + public void run() { + + // 元数据校验 + if (!checkTableMetadata()) { + return; + } + // 初次校验 + firstCheckCompare(); + // 解析初次校验结果 + List diffIdList = parseDiffResult(); + // 根据初次校验结果进行二次校验 + secondaryCheckCompare(diffIdList); + // 校验结果 校验修复报告 + checkResult(); + } + + /** + * 初次校验 + */ + private void firstCheckCompare() { + // 初始化桶列表 + initFirstCheckBucketList(); + compareCommonMerkleTree(); + } + + /** + * 二次校验 + * + * @param diffIdList 初次校验差异ID列表 + */ + private void secondaryCheckCompare(List diffIdList) { + if (CollectionUtils.isEmpty(diffIdList)) { + return; + } + // 清理当前线程捏校验缓存信息 + lastDataClean(); + // 初始化桶列表 + initSecondaryCheckBucketList(diffIdList); + // 进行二次校验 + compareCommonMerkleTree(); + } + + /** + * 初始化桶列表 + */ + private void initFirstCheckBucketList() { + // 获取当前任务对应kafka分区号 + // 初始化源端桶列列表数据 + initFirstCheckBucketList(Endpoint.SOURCE, sourceBucketList); + // 初始化宿端桶列列表数据 + initFirstCheckBucketList(Endpoint.SINK, sinkBucketList); + // 对齐源端宿端桶列表 + alignAllBuckets(); + // 排序 + sortBuckets(sourceBucketList); + sortBuckets(sinkBucketList); + } + + private void initSecondaryCheckBucketList(List diffIdList) { + SourceDataLog dataLog = new SourceDataLog().setTableName(tableName) + .setCompositePrimaryValues(diffIdList); + buildBucket(Endpoint.SOURCE, dataLog); + buildBucket(Endpoint.SINK, dataLog); + // 对齐源端宿端桶列表 + alignAllBuckets(); + // 排序 + sortBuckets(sourceBucketList); + sortBuckets(sinkBucketList); + } + + private void compareCommonMerkleTree() { + //不进行默克尔树校验算法场景 + final int sourceBucketCount = sourceBucketList.size(); + final int sinkBucketCount = sinkBucketList.size(); + if (checkNotMerkleCompare(sourceBucketCount, sinkBucketCount)) { + // 不满足默克尔树约束条件下 比较 sourceSize等于0，即都是空桶 + if (sourceBucketCount == 0) { + //表是空表, 校验成功! + log.info("table[{}] is an empty table,this check successful!", tableName); + } else { + // sourceSize小于thresholdMinBucketSize 即都只有一个桶，比较 + DifferencePair subDifference = compareBucket(sourceBucketList.get(0), sinkBucketList.get(0)); + difference.getDiffering().putAll(subDifference.getDiffering()); + difference.getOnlyOnLeft().putAll(subDifference.getOnlyOnLeft()); + difference.getOnlyOnRight().putAll(subDifference.getOnlyOnRight()); + } + } + + // 构造默克尔树 约束: bucketList 不能为空，且size>=2 + MerkleTree sourceTree = new MerkleTree(sourceBucketList); + MerkleTree sinkTree = new MerkleTree(sinkBucketList); + + //递归比较两颗默克尔树，并将差异记录返回。 + compareMerkleTree(sourceTree, sinkTree); + } + + private void lastDataClean() { + sourceBucketList.clear(); + sinkBucketList.clear(); + difference.getOnlyOnRight().clear(); + difference.getOnlyOnLeft().clear(); + difference.getDiffering().clear(); + } + + + /** + * 根据桶编号对最终桶列表进行排序 + * + * @param bucketList 桶列表 + */ + private void sortBuckets(@NonNull List bucketList) { + bucketList.sort(Comparator.comparingInt(Bucket::getNumber)); + } + + private List parseDiffResult() { + List diffKeyList = new ArrayList<>(); + diffKeyList.addAll(difference.getDiffering().keySet()); + diffKeyList.addAll(difference.getOnlyOnRight().keySet()); + diffKeyList.addAll(difference.getOnlyOnLeft().keySet()); + return diffKeyList; + } + + /** + * 增量校验前置条件，当前表结构一致，若表结构不一致则直接退出。不进行数据校验 + * + * @return 返回元数据校验结果 + */ + private boolean checkTableMetadata() { + TableMetadataHash sourceTableHash = queryTableMetadataHash(Endpoint.SOURCE, tableName); + TableMetadataHash sinkTableHash = queryTableMetadataHash(Endpoint.SINK, tableName); + return Objects.equals(sourceTableHash, sinkTableHash); + } + + private TableMetadataHash queryTableMetadataHash(Endpoint endpoint, String tableName) { + Result result = feignClient.getClient(endpoint).queryTableMetadataHash(tableName); + if (result.isSuccess()) { + return result.getData(); + } else { + throw new DispatchClientException(endpoint, "query table metadata hash " + tableName + + " error, " + result.getMessage()); + } + } + + /** + * 不满足默克尔树约束条件下 比较 + * + * @param sourceBucketCount 源端数量 + * @param sinkBucketCount 宿端数量 + * @return 是否满足默克尔校验场景 + */ + private boolean checkNotMerkleCompare(int sourceBucketCount, int sinkBucketCount) { + // 满足构造默克尔树约束条件 + return sourceBucketCount < THRESHOLD_MIN_BUCKET_SIZE || sinkBucketCount < THRESHOLD_MIN_BUCKET_SIZE; + } + + /** + * 比较两颗默克尔树，并将差异记录返回。 + * + * @param sourceTree 源端默克尔树 + * @param sinkTree 宿端默克尔树 + */ + private void compareMerkleTree(@NonNull MerkleTree sourceTree, @NonNull MerkleTree sinkTree) { + // 默克尔树比较 + if (sourceTree.getDepth() != sinkTree.getDepth()) { + throw new MerkleTreeDepthException(String.format("source & sink data have large different, Please synchronize data again! " + + "merkel tree depth different,source depth=[%d],sink depth=[%d]", + sourceTree.getDepth(), sinkTree.getDepth())); + } + + Node source = sourceTree.getRoot(); + Node sink = sinkTree.getRoot(); + List> diffNodeList = new LinkedList<>(); + compareMerkleTree(source, sink, diffNodeList); + if (CollectionUtils.isEmpty(diffNodeList)) { + return; + } + diffNodeList.forEach(diffNode -> { + Bucket sourceBucket = diffNode.getSource().getBucket(); + Bucket sinkBucket = diffNode.getSink().getBucket(); + DifferencePair subDifference = compareBucket(sourceBucket, sinkBucket); + difference.getDiffering().putAll(subDifference.getDiffering()); + difference.getOnlyOnLeft().putAll(subDifference.getOnlyOnLeft()); + difference.getOnlyOnRight().putAll(subDifference.getOnlyOnRight()); + }); + } + + /** + * 根据统计的源端宿端桶差异信息{@code bucketNumberDiffMap}结果，对齐桶列表数据。 + */ + private void alignAllBuckets() { + dataCheckWapper.alignAllBuckets(bucketNumberDiffMap, sourceBucketList, sinkBucketList); + } + + /** + * 拉取指定端点{@code endpoint}服务当前表{@code tableName}的kafka分区{@code partitions}数据。 + * 并将kafka数据分组组装到指定的桶列表{@code bucketList}中 + * + * @param endpoint 端点类型 + * @param bucketList 桶列表 + */ + private void initFirstCheckBucketList(Endpoint endpoint, List bucketList) { + + // 使用feignclient 拉取kafka数据 + List dataList = getTopicPartitionsData(endpoint); + buildBucket(dataList, endpoint, bucketList); + } + + private void buildBucket(List dataList, Endpoint endpoint, List bucketList) { + if (CollectionUtils.isEmpty(dataList)) { + return; + } + Map bucketMap = new HashMap<>(); + BuilderBucketHandler bucketBuilder = new BuilderBucketHandler(bucketCapacity); + + // 拉取的数据进行构建桶列表 + bucketBuilder.builder(dataList, dataList.size(), bucketMap); + // 统计桶列表信息 + bucketNumberStatisticsIncrement(endpoint, bucketMap.keySet()); + bucketList.addAll(bucketMap.values()); + } + + private void buildBucket(Endpoint endpoint, SourceDataLog dataLog) { + final List dataList = getSecondaryCheckRowData(endpoint, dataLog); + buildBucket(dataList, endpoint, sourceBucketList); + } + + /** + * 对各端点构建的桶编号进行统计。统计结果汇总到{@code bucketNumberDiffMap}中。 + *

+ * 默克尔比较算法，需要确保双方桶编号的一致。 + *

+ * 如果一方的桶编号存在缺失，即{@code Pair}中，S或T的值为-1，则需要生成相应编号的空桶。 + * + * @param endpoint 端点 + * @param bucketNumberSet 桶编号 + */ + private void bucketNumberStatisticsIncrement(@NonNull Endpoint endpoint, @NonNull Set bucketNumberSet) { + bucketNumberSet.forEach(bucketNumber -> { + if (!bucketNumberDiffMap.containsKey(bucketNumber)) { + if (Objects.equals(endpoint, Endpoint.SOURCE)) { + bucketNumberDiffMap.put(bucketNumber, Pair.of(bucketNumber, -1)); + } else { + bucketNumberDiffMap.put(bucketNumber, Pair.of(-1, bucketNumber)); + } + } else { + Pair pair = bucketNumberDiffMap.get(bucketNumber); + if (Objects.equals(endpoint, Endpoint.SOURCE)) { + bucketNumberDiffMap.put(bucketNumber, Pair.of(bucketNumber, pair)); + } else { + bucketNumberDiffMap.put(bucketNumber, Pair.of(pair, bucketNumber)); + } + } + }); + } + + /** + * 拉取指定端点{@code endpoint}的表{@code tableName}的 kafka分区{@code partitions}数据 + * + * @param endpoint 端点类型 + * @return 指定表 kafka分区数据 + */ + private List getTopicPartitionsData(Endpoint endpoint) { + return queryRowDataWapper.queryIncrementRowData(endpoint, tableName); + } + + private List getSecondaryCheckRowData(Endpoint endpoint, SourceDataLog dataLog) { + return queryRowDataWapper.queryRowData(endpoint, dataLog); + } + + /** + * 比较两个桶内部记录的差异数据 + *

+ * 差异类型 {@linkplain org.opengauss.datachecker.common.entry.enums.DiffCategory} + * + * @param sourceBucket 源端桶 + * @param sinkBucket 宿端桶 + * @return 差异记录 + */ + private DifferencePair compareBucket(Bucket sourceBucket, Bucket sinkBucket) { + + Map sourceMap = sourceBucket.getBucket(); + Map sinkMap = sinkBucket.getBucket(); + + MapDifference difference = Maps.difference(sourceMap, sinkMap); + + Map entriesOnlyOnLeft = difference.entriesOnlyOnLeft(); + Map entriesOnlyOnRight = difference.entriesOnlyOnRight(); + Map> entriesDiffering = difference.entriesDiffering(); + Map> differing = new HashMap<>(); + entriesDiffering.forEach((key, diff) -> { + differing.put(key, Pair.of(diff.leftValue(), diff.rightValue())); + }); + return DifferencePair.of(entriesOnlyOnLeft, entriesOnlyOnRight, differing); + } + + /** + * 递归比较两颗默克尔树节点，并记录差异节点。 + *

+ * 采用递归-前序遍历方式，遍历比较默克尔树，从而查找差异节点。 + *

+ * 若当前遍历的节点{@link Node}签名相同则终止当前遍历分支。 + * + * @param source 源端默克尔树节点 + * @param sink 宿端默克尔树节点 + * @param diffNodeList 差异节点记录 + */ + private void compareMerkleTree(@NonNull Node source, @NonNull Node sink, List> diffNodeList) { + // 如果节点相同，则退出 + if (Arrays.equals(source.getSignature(), sink.getSignature())) { + return; + } + // 如果节点不相同，则继续比较下层节点，若当前差异节点为叶子节点，则记录该差异节点，并退出 + if (source.getType() == MerkleTree.LEAF_SIG_TYPE) { + diffNodeList.add(Pair.of(source, sink)); + return; + } + compareMerkleTree(source.getLeft(), sink.getLeft(), diffNodeList); + + compareMerkleTree(source.getRight(), sink.getRight(), diffNodeList); + } + + private void checkResult() { + CheckDiffResult result = AbstractCheckDiffResultBuilder.builder(feignClient) + .table(tableName) + .topic(topic.getTopicName()) + .partitions(partitions) + .keyUpdateSet(difference.getDiffering().keySet()) + .keyInsertSet(difference.getOnlyOnRight().keySet()) + .keyDeleteSet(difference.getOnlyOnLeft().keySet()) + .build(); + ExportCheckResult.export(path, result); + } + + /** + * 重置当前线程 线程名称 + */ + private void resetThreadName() { + Thread.currentThread().setName(THREAD_NAME_PRIFEX + topic.getTopicName()); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/QueryRowDataWapper.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/QueryRowDataWapper.java new file mode 100644 index 0000000..e34fca2 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/check/QueryRowDataWapper.java @@ -0,0 +1,73 @@ +package org.opengauss.datachecker.check.modules.check; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.check.client.FeignClientService; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.RowDataHash; +import org.opengauss.datachecker.common.entry.extract.SourceDataLog; +import org.opengauss.datachecker.common.exception.DispatchClientException; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/18 + * @since ：11 + */ +@Slf4j +public class QueryRowDataWapper { + + private final FeignClientService feignClient; + + public QueryRowDataWapper(FeignClientService feignClient) { + this.feignClient = feignClient; + } + + + /** + * 拉取指定端点{@code endpoint}的表{@code tableName}的 kafka分区{@code partitions}数据 + * + * @param endpoint 端点类型 + * @param partitions kafka分区号 + * @return 指定表 kafka分区数据 + */ + public List queryRowData(Endpoint endpoint, String tableName, int partitions) { + List data = new ArrayList<>(); + Result> result = feignClient.getClient(endpoint).queryTopicData(tableName, partitions); + if (!result.isSuccess()) { + throw new DispatchClientException(endpoint, "query topic data of tableName " + tableName + + " partitions=" + partitions + " error, " + result.getMessage()); + } + while (result.isSuccess() && !CollectionUtils.isEmpty(result.getData())) { + data.addAll(result.getData()); + result = feignClient.getClient(endpoint).queryTopicData(tableName, partitions); + } + return data; + } + + public List queryIncrementRowData(Endpoint endpoint, String tableName) { + List data = new ArrayList<>(); + Result> result = feignClient.getClient(endpoint).queryIncrementTopicData(tableName); + if (!result.isSuccess()) { + throw new DispatchClientException(endpoint, "query topic data of tableName " + tableName + + " error, " + result.getMessage()); + } + while (result.isSuccess() && !CollectionUtils.isEmpty(result.getData())) { + data.addAll(result.getData()); + result = feignClient.getClient(endpoint).queryIncrementTopicData(tableName); + } + return data; + } + + public List queryRowData(Endpoint endpoint, SourceDataLog dataLog) { + Result> result = feignClient.getClient(endpoint).querySecondaryCheckRowData(dataLog); + if (!result.isSuccess()) { + throw new DispatchClientException(endpoint, "query topic data of tableName " + dataLog.getTableName() + + " error, " + result.getMessage()); + } + return result.getData(); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/merkle/MerkleTree.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/merkle/MerkleTree.java new file mode 100644 index 0000000..21d62c6 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/merkle/MerkleTree.java @@ -0,0 +1,308 @@ +package org.opengauss.datachecker.check.modules.merkle; + +import lombok.Data; +import lombok.experimental.Accessors; +import org.opengauss.datachecker.check.modules.bucket.Bucket; +import org.opengauss.datachecker.common.util.ByteUtil; + +import java.nio.ByteBuffer; +import java.util.*; +import java.util.zip.Adler32; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +@Data +public class MerkleTree { + + public static final int MAGIC_HDR = 0Xcdaace99; + private static final int INT_BYTE = 4; + public static final int LONG_BYTE = 8; + /** + * 默克尔树 节点类型 叶子节点 + */ + public static final byte LEAF_SIG_TYPE = 0x0; + /** + * 默克尔树 节点类型 内部节点 + */ + public static final byte INTERNAL_SIG_TYPE = 0x01; + /** + * Serialization format ：(magic header:int)(num nodes:int)(tree depth:int)(leaf length:int) + * [(node type:byte)(signature length:int)(signature:byte)] + */ + private static final int MAGIC_HEADER_BYTE_LENGTH = INT_BYTE; + private static final int LEAF_SIGNATURE_BYTE_LENGTH = INT_BYTE; + private static final int NUM_NODES_BYTE_LENGTH = INT_BYTE; + private static final int TREE_DEPTH_BYTE_LENGTH = INT_BYTE; + private static final int NODE_TYPE_BYTE_LENGTH = 1; + + /** + * Adler32 进行校验 + */ + private volatile static Adler32 crc; + + static { + crc = new Adler32(); + } + + /** + * 叶子节点字节长度 默克尔树在序列化及反序列化时使用。 + */ + private int leafSignatureByteLength; + /** + * 根节点 + */ + private Node root; + private int depth; + private int nnodes; + + /** + * 默克尔树构造函数 + * + * @param bucketList 桶列表 + */ + public MerkleTree(List bucketList) { + constructTree(bucketList); + } + + + /** + * 根据反序列化结果构造默克尔树 + * + * @param treeRoot 树根节点 + * @param totalNodes 总节点数 + * @param depth 树深度 + * @param leafLength 叶子节点长度 + */ + public MerkleTree(Node treeRoot, int totalNodes, int depth, int leafLength) { + this.root = treeRoot; + this.nnodes = totalNodes; + this.depth = depth; + this.leafSignatureByteLength = leafLength; + } + + + /** + * 构造默克尔树 + * + * @param bucketList 桶列表 + */ + private void constructTree(List bucketList) { + if (bucketList == null || bucketList.size() < MerkleConstant.CONSTRUCT_TREE_MIN_SIZE) { + throw new IllegalArgumentException("ERROR:Fail to construct merkle tree ! leafHashes data invalid !"); + } + this.nnodes = bucketList.size(); + List parents = buttomLevel(bucketList); + this.nnodes += parents.size(); + this.depth = 1; + while (parents.size() > 1) { + parents = constructInternalLevel(parents); + this.depth++; + this.nnodes += parents.size(); + } + this.root = parents.get(0); + } + + + /** + * 底部层级叶子节点构建 + * + * @param bucketList 桶列表 + * @return 节点列表 + */ + private List buttomLevel(List bucketList) { + List parents = new ArrayList<>(bucketList.size() / MerkleConstant.EVEN_NUMBER); + for (int i = 0; i < bucketList.size() - 1; i = i + MerkleConstant.EVEN_NUMBER) { + Node leaf1 = constructLeafNode(bucketList.get(i)); + Node leaf2 = constructLeafNode(bucketList.get(i + 1)); + + Node parent = constructInternalNode(leaf1, leaf2); + parents.add(parent); + } + if (bucketList.size() % MerkleConstant.EVEN_NUMBER == 1) { + Node leaf1 = constructLeafNode(bucketList.get(bucketList.size() - 1)); + // 奇数个节点的情况，复制最后一个节点 + Node parent = constructInternalNode(leaf1, null); + parents.add(parent); + } + // 设置叶子节点签名字节长度 + this.leafSignatureByteLength = parents.get(0).getLeft().getSignature().length; + return parents; + } + + /** + * 内部节点构建 + * + * @param children 子节点 + * @return 内部节点集合 + */ + private List constructInternalLevel(List children) { + List parents = new ArrayList<>(children.size() / MerkleConstant.EVEN_NUMBER); + for (int i = 0; i < children.size() - 1; i = i + MerkleConstant.EVEN_NUMBER) { + Node parent = constructInternalNode(children.get(i), children.get(i + 1)); + parents.add(parent); + } + + if (children.size() % MerkleConstant.EVEN_NUMBER == 1) { + // 奇数个节点的情况，只对left节点进行计算 + Node parent = constructInternalNode(children.get(children.size() - 1), null); + parents.add(parent); + } + return parents; + } + + /** + * 构建叶子节点 + * + * @param bucket 桶节点 + * @return 默克尔节点 + */ + private Node constructLeafNode(Bucket bucket) { + return new Node().setType(LEAF_SIG_TYPE) + .setBucket(bucket) + .setSignature(bucket.getSignature()); + } + + /** + * 构建内部节点 + * + * @param left 左侧节点 + * @param right 右侧节点 + * @return 默克尔节点 + */ + private Node constructInternalNode(Node left, Node right) { + return new Node().setType(INTERNAL_SIG_TYPE) + .setLeft(left) + .setRight(right) + .setSignature(internalSignature(left, right)); + } + + /** + * 计算内部节点签名 + * + * @param left 左侧节点 + * @param right 右侧节点 + * @return 内部节点签名 + */ + private byte[] internalSignature(Node left, Node right) { + if (right == null) { + return left.getSignature(); + } + // 这里采用Deler32进行签名 + crc.reset(); + crc.update(left.signature); + crc.update(right.signature); + return ByteUtil.toBytes(crc.getValue()); + } + + /** + * Serialization format ： + * header (magic header:int)(num nodes:int)(tree depth:int)(leaf length:int) + * [(node type:byte)(signature length:int)(signature:byte)(bucket length:int)(bucket:byte)] + *

+ * bucket 桶序列化实现 + * + * @return 返回序列化字节流 + */ + public byte[] serialize() { + int header = MAGIC_HEADER_BYTE_LENGTH + NUM_NODES_BYTE_LENGTH + TREE_DEPTH_BYTE_LENGTH + LEAF_SIGNATURE_BYTE_LENGTH; + int maxSignatureByteLength = Math.max(leafSignatureByteLength, LONG_BYTE); + + int spaceOfNodes = (NODE_TYPE_BYTE_LENGTH + NUM_NODES_BYTE_LENGTH + maxSignatureByteLength) * nnodes; + + int capacity = header + spaceOfNodes; + ByteBuffer buffer = ByteBuffer.allocate(capacity); + //header (magic header:int)(num nodes:int)(tree depth:int)(leaf length:int) + buffer.putInt(MAGIC_HDR).putInt(nnodes).putInt(depth).putInt(leafSignatureByteLength); + serializeNode(buffer); + + byte[] serializeTree = new byte[buffer.position()]; + buffer.rewind(); + buffer.get(serializeTree); + return serializeTree; + } + + private void serializeNode(ByteBuffer buffer) { + Queue queue = new ArrayDeque<>(nnodes / 2 + 1); + queue.add(root); + while (!queue.isEmpty()) { + Node node = queue.remove(); + buffer.put(node.type).putInt(node.signature.length).put(node.signature); + if (node.getLeft() != null) { + queue.add(node.getLeft()); + } + if (node.getRight() != null) { + queue.add(node.getRight()); + } + } + } + + /** + * Merkle Tree 节点 + */ + @Data + @Accessors(chain = true) + public static class Node { + + /** + * LEAF_SIG_TYPE,INTERNAL_SIG_TYPE + */ + private byte type; + private Node left; + private Node right; + /** + * 当前节点签名 signature + */ + private byte[] signature; + private Bucket bucket; + + @Override + public String toString() { + return " Node{" + + "type=" + type + + ",signature=" + Arrays.toString(signature).replace(",", "") + + ",left=" + left + + ",right=" + right + + '}'; + } + + public boolean signatureEqual(Node other) { + int length = this.getSignature().length; + int length1 = other.getSignature().length; + if (length != length1) { + return false; + } + for (int i = 0; i < length; i++) { + if (this.getSignature()[i] != other.getSignature()[i]) { + return false; + } + } + return true; + } + } + + @Override + public String toString() { + return "MerkleTree{" + + "nnodes=" + nnodes + + ",depth=" + depth + + ",leafSignatureByteLength=" + leafSignatureByteLength + + ",root=" + root + + '}'; + } + + public String toSimpleString() { + return "MerkleTree{" + + "nnodes=" + nnodes + + ",depth=" + depth + + ",leafSignatureByteLength=" + leafSignatureByteLength + + '}'; + } + + interface MerkleConstant { + int CONSTRUCT_TREE_MIN_SIZE = 2; + int EVEN_NUMBER = 2; + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/merkle/MerkleTreeDeserializer.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/merkle/MerkleTreeDeserializer.java new file mode 100644 index 0000000..9af70ad --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/merkle/MerkleTreeDeserializer.java @@ -0,0 +1,81 @@ +package org.opengauss.datachecker.check.modules.merkle; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.Queue; + +import net.openhft.hashing.LongHashFunction; +import org.opengauss.datachecker.check.modules.merkle.MerkleTree.Node; +import org.opengauss.datachecker.common.util.ByteUtil; + +/** + * 默克尔树反序列化 + * bucket 桶反序列化实现 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class MerkleTreeDeserializer { + + /** + * 反序列化根据{@link MerkleTree#serialize()}返回的字节数组实现 + * Serialization format ： + * header (magic header:int)(num nodes:int)(tree depth:int)(leaf length:int) + * [(node type:byte)(signature length:int)(signature:byte)] + * + * @param serializerTree + * @return + */ + public static MerkleTree deserialize(byte[] serializerTree) { + ByteBuffer buffer = ByteBuffer.wrap(serializerTree); + + // 字节数组头校验 + if (buffer.getInt() != MerkleTree.MAGIC_HDR) { + throw new IllegalArgumentException("序列化字节数组没有已合法的Magic Header开头"); + } + // 读取头信息 + int totalNodes = buffer.getInt(); + int depth = buffer.getInt(); + int leafLength = buffer.getInt(); + + // 读取 root 节点 + Node root = new Node() + .setType(buffer.get()) + .setSignature(readNextSingature(buffer)); + if (root.getType() == MerkleTree.LEAF_SIG_TYPE) { + throw new IllegalArgumentException("首个序列化节点为叶子节点"); + } + + Queue queue = new ArrayDeque<>(totalNodes / 2 + 1); + + Node currentNode = root; + for (int i = 1; i < totalNodes; i++) { + Node child = new Node() + .setType(buffer.get()) + .setSignature(readNextSingature(buffer)); + queue.add(child); + // 处理节点已提升的不完整树 : (如果currentNode 和child节点的签名一致) + if (ByteUtil.isEqual(currentNode.getSignature(), child.getSignature())) { + currentNode.setLeft(child); + currentNode = queue.remove(); + continue; + } + if (currentNode.getLeft() == null) { + currentNode.setLeft(child); + } else { + currentNode.setRight(child); + currentNode = queue.remove(); + } + + } + return new MerkleTree(root, totalNodes, depth, leafLength); + } + + + private static byte[] readNextSingature(ByteBuffer buffer) { + byte[] singatureBytes = new byte[buffer.getInt()]; + buffer.get(singatureBytes); + return singatureBytes; + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/task/TaskManagerService.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/task/TaskManagerService.java new file mode 100644 index 0000000..4eadc96 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/task/TaskManagerService.java @@ -0,0 +1,33 @@ +package org.opengauss.datachecker.check.modules.task; + +import org.opengauss.datachecker.common.entry.enums.Endpoint; + +import java.util.List; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/25 + * @since ：11 + */ +public interface TaskManagerService { + /** + * 刷新指定任务的数据抽取表执行状态 + * + * @param tableName 表名称 + * @param endpoint 端点类型 {@link org.opengauss.datachecker.common.entry.enums.Endpoint} + */ + void refushTableExtractStatus(String tableName, Endpoint endpoint); + + + /** + * 初始化任务状态 + * + * @param tableNameList 表名称列表 + */ + void initTableExtractStatus(List tableNameList); + + /** + * 清理任务状态信息 + */ + void cleanTaskStatus(); +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/task/TaskManagerServiceImpl.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/task/TaskManagerServiceImpl.java new file mode 100644 index 0000000..2a03eff --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/modules/task/TaskManagerServiceImpl.java @@ -0,0 +1,63 @@ +package org.opengauss.datachecker.check.modules.task; + +import com.alibaba.fastjson.JSON; +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.check.cache.TableStatusRegister; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.exception.CheckingException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.HashSet; +import java.util.List; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/25 + * @since ：11 + */ +@Slf4j +@Service +public class TaskManagerServiceImpl implements TaskManagerService { + + @Autowired + private TableStatusRegister tableStatusRegister; + + /** + * 刷新指定任务的数据抽取表执行状态 + * + * @param tableName 表名称 + * @param endpoint 端点类型 {@link org.opengauss.datachecker.common.entry.enums.Endpoint} + */ + @Override + public void refushTableExtractStatus(String tableName, Endpoint endpoint) { + log.info("check server refush endpoint=[{}] extract tableName=[{}] status=[{}] ", endpoint.getDescription(), tableName, endpoint.getCode()); + tableStatusRegister.update(tableName, endpoint.getCode()); + } + + /** + * 初始化任务状态 + * + * @param tableNameList 表名称列表 + */ + @Override + public void initTableExtractStatus(List tableNameList) { + if (tableStatusRegister.isEmpty() || tableStatusRegister.isCheckComplated()) { + cleanTaskStatus(); + tableStatusRegister.init(new HashSet<>(tableNameList)); + log.info("check server init extract tableNameList=[{}] status= ", JSON.toJSONString(tableNameList)); + } else { + //上次校验流程正在执行，不能重新初始化表校验状态数据！ + throw new CheckingException("The last verification process is being executed, and the table verification status data cannot be reinitialized!"); + } + + } + + /** + * 清理任务状态信息 + */ + @Override + public void cleanTaskStatus() { + tableStatusRegister.removeAll(); + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckBlackWhiteService.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckBlackWhiteService.java new file mode 100644 index 0000000..5f94250 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckBlackWhiteService.java @@ -0,0 +1,105 @@ +package org.opengauss.datachecker.check.service; + +import org.opengauss.datachecker.check.client.FeignClientService; +import org.opengauss.datachecker.check.config.DataCheckProperties; +import org.opengauss.datachecker.common.entry.enums.CheckBlackWhiteMode; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListSet; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/22 + * @since ：11 + */ +@Service +public class CheckBlackWhiteService { + private static final Set WHITE = new ConcurrentSkipListSet<>(); + private static final Set BLACK = new ConcurrentSkipListSet<>(); + + @Autowired + private FeignClientService feignClientService; + + @Autowired + private DataCheckProperties dataCheckProperties; + + /** + * 添加白名单列表 该功能清理历史白名单，重置白名单为当前列表 + * + * @param whiteList 白名单列表 + */ + public void addWhiteList(List whiteList) { + WHITE.clear(); + WHITE.addAll(whiteList); + refushWhiteList(); + } + + /** + * 更新白名单列表 该功能在当前白名单基础上新增当前列表到白名单 + * + * @param whiteList 白名单列表 + */ + public void updateWhiteList(List whiteList) { + WHITE.addAll(whiteList); + refushWhiteList(); + } + + /** + * 移除白名单列表 该功能在当前白名单基础上移除当前列表到白名单 + * + * @param whiteList 白名单列表 + */ + public void deleteWhiteList(List whiteList) { + WHITE.removeAll(whiteList); + refushWhiteList(); + } + + /** + * 查询白名单列表 + * + * @return 白名单列表 + */ + public List queryWhiteList() { + return new ArrayList<>(WHITE); + } + + public void addBlackList(List blackList) { + BLACK.clear(); + BLACK.addAll(blackList); + refushWhiteList(); + } + + public void updateBlackList(List blackList) { + BLACK.addAll(blackList); + refushWhiteList(); + } + + public void deleteBlackList(List blackList) { + BLACK.removeAll(blackList); + refushWhiteList(); + } + + public List queryBlackList() { + return new ArrayList<>(BLACK); + } + + private void refushWhiteList() { + final CheckBlackWhiteMode blackWhiteMode = dataCheckProperties.getBlackWhiteMode(); + if (blackWhiteMode == CheckBlackWhiteMode.WHITE) { + // 白名单模式 + feignClientService.getClient(Endpoint.SOURCE).refushBlackWhiteList(blackWhiteMode, new ArrayList<>(WHITE)); + feignClientService.getClient(Endpoint.SINK).refushBlackWhiteList(blackWhiteMode, new ArrayList<>(WHITE)); + } else if (blackWhiteMode == CheckBlackWhiteMode.BLACK) { + // 黑名单模式 + feignClientService.getClient(Endpoint.SOURCE).refushBlackWhiteList(blackWhiteMode, new ArrayList<>(BLACK)); + feignClientService.getClient(Endpoint.SINK).refushBlackWhiteList(blackWhiteMode, new ArrayList<>(BLACK)); + } + } + + +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckService.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckService.java new file mode 100644 index 0000000..bfb4767 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/CheckService.java @@ -0,0 +1,40 @@ +package org.opengauss.datachecker.check.service; + + +import org.opengauss.datachecker.common.entry.check.IncrementCheckConifg; +import org.opengauss.datachecker.common.entry.enums.CheckMode; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +public interface CheckService { + + /** + * 开启校验服务 + * + * @param checkMode 校验方式 + * @return 进程号 + */ + String start(CheckMode checkMode); + + /** + * 查询当前执行的进程号 + * + * @return 进程号 + */ + String getCurrentCheckProcess(); + + /** + * 清理校验环境 + */ + void cleanCheck(); + + /** + * 增量校验配置初始化 + * + * @param incrementCheckConifg 初始化配置 + */ + void incrementCheckConifg(IncrementCheckConifg incrementCheckConifg); +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/EndpointManagerService.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/EndpointManagerService.java new file mode 100644 index 0000000..638ee82 --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/EndpointManagerService.java @@ -0,0 +1,140 @@ +package org.opengauss.datachecker.check.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.opengauss.datachecker.check.client.FeignClientService; +import org.opengauss.datachecker.check.config.DataCheckProperties; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.util.concurrent.*; + +/** + * 数据抽取服务端点管理 + * + * @author ：wangchao + * @date ：Created in 2022/5/26 + * @since ：11 + */ +@Slf4j +@Service +public class EndpointManagerService { + + private static final String ENDPOINT_HEALTH_CHECK_THREAD_NAME = "endpoint-health-check-thread"; + private final ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); + + @Autowired + private FeignClientService feignClientService; + + @Autowired + private DataCheckProperties dataCheckProperties; + + @PostConstruct + public void start() { + scheduledExecutor.scheduleWithFixedDelay(() -> { + Thread.currentThread().setName(ENDPOINT_HEALTH_CHECK_THREAD_NAME); + endpointHealthCheck(); + }, 0, 5, TimeUnit.SECONDS); + } + + public void endpointHealthCheck() { + checkEndpoint(dataCheckProperties.getSourceUri(), Endpoint.SOURCE, "源端服务检查"); + checkEndpoint(dataCheckProperties.getSinkUri(), Endpoint.SINK, "目标端服务检查"); + } + + private void checkEndpoint(String requestUri, Endpoint endpoint, String message) { + // 服务网络检查ping + try { + if (NetworkCheck.networkCheck(getEndpointIp(requestUri))) { + + // 服务检查 服务数据库检查 + Result healthStatus = feignClientService.getClient(endpoint).health(); + if (healthStatus.isSuccess()) { + log.debug("{}：{} current state health", message, requestUri); + } else { + log.error("{}:{} current service status is abnormal", message, requestUri); + } + + } + } catch (Exception ce) { + log.error("{}:{} service unreachable", message, ce.getMessage()); + } + } + + /** + * 根据配置属性中的端点URI地址，解析对应的IP地址 + * URI地址: http://127.0.0.1:8080 https://127.0.0.1:8080 + * + * @param endpointUri 配置属性中的端点URI + * @return 若解析成功，则返回对应IP地址，否则返回null + */ + private String getEndpointIp(String endpointUri) { + if ((endpointUri.contains(NetAddress.HTTP) || endpointUri.contains(NetAddress.HTTPS)) + && endpointUri.contains(NetAddress.IP_DELEMTER) && endpointUri.contains(NetAddress.PORT_DELEMTER)) { + return endpointUri.replace(NetAddress.IP_DELEMTER, NetAddress.PORT_DELEMTER).split(NetAddress.PORT_DELEMTER)[1]; + } + return null; + } + + interface NetAddress { + String HTTP = "http"; + String HTTPS = "https"; + String IP_DELEMTER = "://"; + String PORT_DELEMTER = ":"; + } + + /** + * 网络状态检查 + */ + static class NetworkCheck { + private static final String PING = "ping "; + private static final String TTL = "TTL"; + + /** + * 根据系统命令 ping {@code ip} 检查网络状态 + * + * @param ip ip 地址 + * @return 网络检查结果 + */ + public static boolean networkCheck(String ip) { + boolean result = false; + if (StringUtils.isEmpty(ip)) { + log.error("network check error : ip addr is null"); + return result; + } + + String line; + String endMsg = null; + StringBuffer sb = new StringBuffer(); + String cmd = PING + ip; + try { + Process process = Runtime.getRuntime().exec(cmd); + try (BufferedReader buffer = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName("GBK")))) { + while ((line = buffer.readLine()) != null) { + sb.append(line); + endMsg = line; + } + if (StringUtils.contains(sb.toString(), TTL)) { + result = true; + log.debug("ip {} network check normal", cmd); + } else { + log.error("ip {} network check error : {}", cmd, endMsg); + } + } catch (IOException io) { + throw new IOException("read process result bufferedReader error"); + } + } catch (IOException io) { + log.error("ip {} network check error : {} ", ip, io.getMessage()); + } + return result; + } + } + +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/IncrementManagerService.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/IncrementManagerService.java new file mode 100644 index 0000000..30d62ba --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/IncrementManagerService.java @@ -0,0 +1,46 @@ +package org.opengauss.datachecker.check.service; + +import org.opengauss.datachecker.check.client.FeignClientService; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.SourceDataLog; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/14 + * @since ：11 + */ +@Service +public class IncrementManagerService { + + @Autowired + private FeignClientService feignClientService; + + /** + * 增量校验日志通知 + * + * @param dataLogList 增量校验日志 + */ + public void notifySourceIncrementDataLogs(List dataLogList) { + // 收集上次校验结果，并构建增量校验日志 + dataLogList.addAll(collectLastResults()); + + feignClientService.notifyIncrementDataLogs(Endpoint.SOURCE, dataLogList); + feignClientService.notifyIncrementDataLogs(Endpoint.SINK, dataLogList); + } + + /** + * 收集上次校验结果，并构建增量校验日志 + * + * @return 上次校验结果解析 + */ + private List collectLastResults() { + List dataLogList = new ArrayList<>(); + + return dataLogList; + } +} diff --git a/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/impl/CheckServiceImpl.java b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/impl/CheckServiceImpl.java new file mode 100644 index 0000000..c33731a --- /dev/null +++ b/datachecker-check/src/main/java/org/opengauss/datachecker/check/service/impl/CheckServiceImpl.java @@ -0,0 +1,248 @@ +package org.opengauss.datachecker.check.service.impl; + +import com.alibaba.fastjson.JSON; +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.check.cache.TableStatusRegister; +import org.opengauss.datachecker.check.client.FeignClientService; +import org.opengauss.datachecker.check.modules.check.DataCheckService; +import org.opengauss.datachecker.check.service.CheckService; +import org.opengauss.datachecker.common.entry.check.IncrementCheckConifg; +import org.opengauss.datachecker.common.entry.enums.CheckMode; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.ExtractTask; +import org.opengauss.datachecker.common.entry.extract.Topic; +import org.opengauss.datachecker.common.exception.CheckingException; +import org.opengauss.datachecker.common.exception.CheckingPollingException; +import org.opengauss.datachecker.common.util.IdWorker; +import org.opengauss.datachecker.common.util.ThreadUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +@Slf4j +@Service(value = "checkService") +public class CheckServiceImpl implements CheckService { + /** + * 校验任务启动标志 + *

+ * 无论是全量校验和增量校验，同一时间内只能执行一个。 + * 只有本地全量或者增量校验执行完成后，即{@code STARTED}==false时，才可以执行下一个。 + * 否则直接退出，等待当前校验流程执行完毕，自动退出。 + *

+ * 暂时不提供强制退出当前校验流程方法。 + */ + private static final AtomicBoolean STARTED = new AtomicBoolean(false); + + /** + * 进程签名 后期是否删除进程签名逻辑 + */ + @Deprecated + private static final AtomicReference PROCESS_SIGNATURE = new AtomicReference<>(); + + /** + * 校验模式 + */ + private static final AtomicReference CHECK_MODE_REF = new AtomicReference<>(); + + /** + * 校验轮询线程名称 + */ + private static final String SELF_CHECK_POLL_THREAD_NAME = "check-polling-thread"; + + /** + * 单线程定时任务 - 执行校验轮询线程 Thread.name={@value SELF_CHECK_POLL_THREAD_NAME} + */ + private final ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); + + private final ThreadPoolExecutor singleThreadExecutor = ThreadUtil.newSingleThreadExecutor(); + + @Autowired + private FeignClientService feignClientService; + + @Autowired + private TableStatusRegister tableStatusRegister; + + @Resource + private DataCheckService dataCheckService; + + /** + * 开启校验服务 + * + * @param checkMode 校验方式 + */ + @Override + public String start(CheckMode checkMode) { + if (STARTED.compareAndSet(false, true)) { + log.info("check service is starting, start check mode is [{}]", checkMode.getCode()); + CHECK_MODE_REF.set(checkMode); + if (Objects.equals(CheckMode.FULL, checkMode)) { + startCheckFullMode(); + // 等待任务构建完成，开启任务轮询线程 + startCheckPollingThread(); + } else { + startCheckIncrementMode(); + } + } else { + String message = String.format("check service is running, current check mode is [%s] , exit.", checkMode.getDescription()); + log.error(message); + throw new CheckingException(message); + } + return PROCESS_SIGNATURE.get(); + } + + /** + * 开启全量校验模式 + */ + private void startCheckFullMode() { + String processNo = IdWorker.nextId36(); + // 元数据信息查询 + feignClientService.queryMetaDataOfSchema(Endpoint.SOURCE); + feignClientService.queryMetaDataOfSchema(Endpoint.SINK); + log.info("check full mode : query meta data from db schema (source and sink )"); + // 源端任务构建 + final List extractTasks = feignClientService.buildExtractTaskAllTables(Endpoint.SOURCE, processNo); + extractTasks.forEach(task -> log.debug("check full mode : build extract task source {} : {}", processNo, JSON.toJSONString(task))); + // 宿端任务构建 + feignClientService.buildExtractTaskAllTables(Endpoint.SINK, processNo, extractTasks); + log.info("check full mode : build extract task sink {}", processNo); + // 构建任务执行 + feignClientService.execExtractTaskAllTables(Endpoint.SOURCE, processNo); + feignClientService.execExtractTaskAllTables(Endpoint.SINK, processNo); + log.info("check full mode : exec extract task (source and sink ) {}", processNo); + PROCESS_SIGNATURE.set(processNo); + } + + /** + * /** + * 数据校验轮询线程 + * 用于实时监测数据抽取任务的完成状态。 + * 当某一数据抽取任务状态变更为完成时，启动一个数据校验独立线程。并开启当前任务，进行数据校验。 + */ + public void startCheckPollingThread() { + if (Objects.nonNull(PROCESS_SIGNATURE.get()) && Objects.equals(CHECK_MODE_REF.getAcquire(), CheckMode.FULL)) { + scheduledExecutor.scheduleWithFixedDelay(() -> { + Thread.currentThread().setName(SELF_CHECK_POLL_THREAD_NAME); + log.debug("check polling processNo={}", PROCESS_SIGNATURE.get()); + if (Objects.isNull(PROCESS_SIGNATURE.get())) { + throw new CheckingPollingException("process is empty,stop check polling"); + } + // 是否有表数据抽取完成 + if (tableStatusRegister.hasExtractComplated()) { + // 获取数据抽取完成表名 + String tableName = tableStatusRegister.complatedTablePoll(); + if (Objects.isNull(tableName)) { + return; + } + Topic topic = feignClientService.queryTopicInfo(Endpoint.SOURCE, tableName); + + if (Objects.nonNull(topic)) { + IntStream.range(0, topic.getPartitions()).forEach(idxPartition -> { + log.info("kafka consumer topic=[{}] partitions=[{}]", topic.toString(), idxPartition); + // 根据表名称 和kafka分区进行数据校验 + dataCheckService.checkTableData(topic, idxPartition); + }); + } + complateProgressBar(); + } + }, 0, 1, TimeUnit.SECONDS); + } + } + + + private void complateProgressBar() { + singleThreadExecutor.submit(() -> { + Thread.currentThread().setName("complated-process-bar"); + int total = tableStatusRegister.getKeys().size(); + int complated = tableStatusRegister.complateSize(); + log.info("current check process has task total=[{}] , complate=[{}]", total, complated); + }); + } + + /** + * 开启增量校验模式 + */ + private void startCheckIncrementMode() { + // 开启增量校验模式-轮询线程启动 + if (Objects.equals(CHECK_MODE_REF.getAcquire(), CheckMode.INCREMENT)) { + scheduledExecutor.scheduleWithFixedDelay(() -> { + Thread.currentThread().setName(SELF_CHECK_POLL_THREAD_NAME); + log.debug("check polling check mode=[{}]", CHECK_MODE_REF.get()); + // 是否有表数据抽取完成 + if (tableStatusRegister.hasExtractComplated()) { + // 获取数据抽取完成表名 + String tableName = tableStatusRegister.complatedTablePoll(); + if (Objects.isNull(tableName)) { + return; + } + Topic topic = feignClientService.getIncrementTopicInfo(Endpoint.SOURCE, tableName); + + if (Objects.nonNull(topic)) { + log.info("kafka consumer topic=[{}]", topic.toString()); + // 根据表名称 和kafka分区进行数据校验 + dataCheckService.incrementCheckTableData(topic); + } + complateProgressBar(); + } + // 当前周期任务完成校验，重置任务状态 + if (tableStatusRegister.isCheckComplated()) { + log.info("当前周期校验完成，重置任务状态！"); + tableStatusRegister.rest(); + feignClientService.cleanTask(Endpoint.SOURCE); + feignClientService.cleanTask(Endpoint.SINK); + } + }, 0, 1, TimeUnit.SECONDS); + } + } + + /** + * 查询当前执行的进程号 + * + * @return 进程号 + */ + @Override + public String getCurrentCheckProcess() { + return PROCESS_SIGNATURE.get(); + } + + /** + * 清理校验环境 + */ + @Override + public synchronized void cleanCheck() { + cleanBuildedTask(); + ThreadUtil.sleep(3000); + CHECK_MODE_REF.set(null); + PROCESS_SIGNATURE.set(null); + STARTED.set(false); + log.info("清除当前校验服务标识！"); + log.info("重置校验服务启动标识！"); + } + + @Override + public void incrementCheckConifg(IncrementCheckConifg incrementCheckConifg) { + feignClientService.configIncrementCheckEnvironment(Endpoint.SOURCE, incrementCheckConifg); + } + + private void cleanBuildedTask() { + try { + feignClientService.cleanEnvironment(Endpoint.SOURCE, PROCESS_SIGNATURE.get()); + feignClientService.cleanEnvironment(Endpoint.SINK, PROCESS_SIGNATURE.get()); + } catch (RuntimeException ex) { + log.error("ignore error:", ex); + } + tableStatusRegister.removeAll(); + log.info("数据抽取任务清除 "); + } +} diff --git a/datachecker-check/src/main/resources/application.yml b/datachecker-check/src/main/resources/application.yml new file mode 100644 index 0000000..6b7412d --- /dev/null +++ b/datachecker-check/src/main/resources/application.yml @@ -0,0 +1,45 @@ +server: + port: 7000 + +debug: false + +spring: + application: + name: datachecker-check + datasource: + druid: + dataCheck: + driver-class-name: com.mysql.cj.jdbc.Driver + type: com.alibaba.druid.pool.DruidDataSource + #Spring Boot 默认是不注入这些属性值的，需要自己绑定 + #druid 数据源专有配置 + initialSize: 5 + minIdle: 5 + maxActive: 20 + maxWait: 60000 + timeBetweenEvictionRunsMillis: 60000 + minEvictableIdleTimeMillis: 300000 + validationQuery: SELECT 1 FROM DUAL + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + poolPreparedStatements: true + + +fegin: + hystrix: + enabled:true + +logging: + config: classpath:log4j2.xml + +data: + check: + data-path: local_path/xxx # 配置数据校验结果输出本地路径 + bucket-expect-capacity: 10 # 桶容量范围最小值为1 + health-check-api: /extract/health + black-white-mode: BLACK #大写 + + + + diff --git a/datachecker-check/src/main/resources/log4j2.xml b/datachecker-check/src/main/resources/log4j2.xml new file mode 100644 index 0000000..a9dd5a2 --- /dev/null +++ b/datachecker-check/src/main/resources/log4j2.xml @@ -0,0 +1,123 @@ + + + + + + + logs/check + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/datachecker-check/src/test/java/org/opengauss/datachecker/check/config/DataCheckConfigTest.java b/datachecker-check/src/test/java/org/opengauss/datachecker/check/config/DataCheckConfigTest.java new file mode 100644 index 0000000..8db6055 --- /dev/null +++ b/datachecker-check/src/test/java/org/opengauss/datachecker/check/config/DataCheckConfigTest.java @@ -0,0 +1,21 @@ +package org.opengauss.datachecker.check.config; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class DataCheckConfigTest { + + + @Autowired + private DataCheckConfig dataCheckConfig; + + + @Test + void testGetCheckResultPaht() { + + final String checkResultPaht = dataCheckConfig.getCheckResultPath(); + System.out.println(checkResultPaht); + } +} diff --git a/datachecker-check/src/test/java/org/opengauss/datachecker/check/controller/TaskStatusControllerTest.java b/datachecker-check/src/test/java/org/opengauss/datachecker/check/controller/TaskStatusControllerTest.java new file mode 100644 index 0000000..e392bbf --- /dev/null +++ b/datachecker-check/src/test/java/org/opengauss/datachecker/check/controller/TaskStatusControllerTest.java @@ -0,0 +1,45 @@ +package org.opengauss.datachecker.check.controller; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.opengauss.datachecker.check.modules.task.TaskManagerService; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.web.servlet.MockMvc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; + +@ExtendWith(SpringExtension.class) +@WebMvcTest(TaskStatusController.class) +class TaskStatusControllerTest { + + @Autowired + private MockMvc mockMvc; + + @MockBean + private TaskManagerService taskManagerService; + + @Test + void testRefushTaskExtractStatus() throws Exception { + // Setup + // Run the test + final MockHttpServletResponse response = mockMvc.perform(post("/table/extract/status") + .param("tableName", "tableName") + .param("endpoint", Endpoint.SOURCE.name()) + .accept(MediaType.APPLICATION_JSON)) + .andReturn().getResponse(); + + // Verify the results + assertThat(response.getStatus()).isEqualTo(HttpStatus.OK.value()); + assertThat(response.getContentAsString()).isEqualTo(""); + verify(taskManagerService).refushTableExtractStatus("tableName", Endpoint.SOURCE); + } +} diff --git a/datachecker-check/src/test/java/org/opengauss/datachecker/check/modules/bucket/TestBucket.java b/datachecker-check/src/test/java/org/opengauss/datachecker/check/modules/bucket/TestBucket.java new file mode 100644 index 0000000..a1ae7da --- /dev/null +++ b/datachecker-check/src/test/java/org/opengauss/datachecker/check/modules/bucket/TestBucket.java @@ -0,0 +1,71 @@ +package org.opengauss.datachecker.check.modules.bucket; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.opengauss.datachecker.common.util.HashUtil; +import org.opengauss.datachecker.common.util.IdWorker; + +import java.util.stream.IntStream; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/10 + * @since ：11 + */ +@Slf4j +public class TestBucket { + private static final int[] BUCKET_COUNT_LIMITS = new int[15]; + /** + * 空桶容量大小，用于构造特殊的空桶 + */ + private static final int EMPTY_INITIAL_CAPACITY = 0; + + private static final int BUCKET_MAX_COUNT_LIMITS = 1 << 15; + + static { + BUCKET_COUNT_LIMITS[0] = 1 << 1; + BUCKET_COUNT_LIMITS[1] = 1 << 2; + BUCKET_COUNT_LIMITS[2] = 1 << 3; + BUCKET_COUNT_LIMITS[3] = 1 << 4; + BUCKET_COUNT_LIMITS[4] = 1 << 5; + BUCKET_COUNT_LIMITS[5] = 1 << 6; + BUCKET_COUNT_LIMITS[6] = 1 << 7; + BUCKET_COUNT_LIMITS[7] = 1 << 8; + BUCKET_COUNT_LIMITS[8] = 1 << 9; + BUCKET_COUNT_LIMITS[9] = 1 << 10; + BUCKET_COUNT_LIMITS[10] = 1 << 11; + BUCKET_COUNT_LIMITS[11] = 1 << 12; + BUCKET_COUNT_LIMITS[12] = 1 << 13; + BUCKET_COUNT_LIMITS[13] = 1 << 14; + BUCKET_COUNT_LIMITS[14] = BUCKET_MAX_COUNT_LIMITS; + } + + @Test + public void test() { + IntStream.rangeClosed(0, 14).forEach(idx -> { + System.out.println("1<<" + (idx + 1) + " == " + BUCKET_COUNT_LIMITS[idx]); + }); + + } + + + @Test + public void test2() { + final int limit = BUCKET_COUNT_LIMITS[6]; + IntStream.rangeClosed(0, 14).forEach(idx -> { + final String squeueID = IdWorker.nextId("F"); + final long hashVal = HashUtil.hashBytes(squeueID); + log.info("squeueID[{}] % limit[{}] calacA={}, calacB={}", hashVal, limit, calacA(hashVal, limit), calacB(hashVal, limit)); + }); + + } + + private int calacA(long primaryKeyHash, int bucketCount) { +// return (int) (primaryKeyHash & (bucketCount - 1)); + return (int) (Math.abs(primaryKeyHash) % bucketCount); + } + + private int calacB(long primaryKeyHash, int bucketCount) { + return (int) (Math.abs(primaryKeyHash) & (bucketCount - 1)); + } +} diff --git a/datachecker-check/src/test/java/org/opengauss/datachecker/check/task/TableStatusRegisterTest.java b/datachecker-check/src/test/java/org/opengauss/datachecker/check/task/TableStatusRegisterTest.java new file mode 100644 index 0000000..2648772 --- /dev/null +++ b/datachecker-check/src/test/java/org/opengauss/datachecker/check/task/TableStatusRegisterTest.java @@ -0,0 +1,114 @@ +package org.opengauss.datachecker.check.task; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opengauss.datachecker.check.cache.TableStatusRegister; + +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +class TableStatusRegisterTest { + + private TableStatusRegister tableStatusRegisterUnderTest; + + @BeforeEach + void setUp() { + tableStatusRegisterUnderTest = new TableStatusRegister(); + testInit(); + } + + @Test + void testInit() { + // Setup + // Run the test + tableStatusRegisterUnderTest.init(Set.of("tabel1", "tabel2", "tabel3", "tabel4")); + // Verify the results + } + + @Test + void testPut() { + tableStatusRegisterUnderTest.put("tabel5", 3); + } + + @Test + void testGet() { + assertThat(tableStatusRegisterUnderTest.get("tabel1")).isEqualTo(0); + } + + @Test + void testUpdate() { + System.out.println("0|1 = " + (0 | 1)); + System.out.println("0|2 = " + (0 | 2)); + System.out.println("1|1 = " + (1 | 1)); + System.out.println("1|2 = " + (1 | 2)); + System.out.println("1|2|4 = " + (1 | 2 | 4)); + System.out.println("4 = " + Integer.toBinaryString(4)); + System.out.println(tableStatusRegisterUnderTest.get("tabel1")); + assertThat(tableStatusRegisterUnderTest.update("tabel1", 1)).isEqualTo(1); + + } + + @Test + void testRemove() { + // Setup + // Run the test + tableStatusRegisterUnderTest.remove("key"); + + // Verify the results + } + + @Test + void testRemoveAll() { + // Setup + // Run the test + tableStatusRegisterUnderTest.removeAll(); + + // Verify the results + } + + /** + * 线程状态观测 + * Thread.State + * 线程状态。线程可处于以下状态之一： + * NEW 尚未启动的线程处于此状态 + * RUNNABLE 在Java虚拟机中执行的线程处于此状态 + * BLOCKED 被阻塞等待监视器锁定的线程处于此状态 + * WAITING 正在等待另一个线程执行特定的动作的线程处于此状态 + * TIMED_WAITING 正在等待另一个线程执行动作达到指定等待时间的线程处于此状态 + * TERMINATED 已退出的线程处于此状态 + * + * @throws InterruptedException + */ + @Test + void testPersistent() throws InterruptedException { + Thread thread = new Thread(() -> { + for (int i = 0; i < 5; i++) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + System.out.println("------------"); + }); + + Thread.State state = thread.getState(); + System.out.println(state); + + thread.start(); + state = thread.getState(); + System.out.println(state); + + boolean a = true; + while (a) { + Thread.sleep(2000); + System.out.println(thread.getState()); + + thread.start(); + + System.out.println(thread.getState()); + a = false; + } + } +} diff --git a/datachecker-check/src/test/java/org/opengauss/datacheckercheck/DatacheckerCheckApplicationTests.java b/datachecker-check/src/test/java/org/opengauss/datacheckercheck/DatacheckerCheckApplicationTests.java new file mode 100644 index 0000000..4fe7ba6 --- /dev/null +++ b/datachecker-check/src/test/java/org/opengauss/datacheckercheck/DatacheckerCheckApplicationTests.java @@ -0,0 +1,13 @@ +package org.opengauss.datacheckercheck; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class DatacheckerCheckApplicationTests { + + @Test + void contextLoads() { + } + +} diff --git a/datachecker-common/.gitignore b/datachecker-common/.gitignore new file mode 100644 index 0000000..549e00a --- /dev/null +++ b/datachecker-common/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/datachecker-common/pom.xml b/datachecker-common/pom.xml new file mode 100644 index 0000000..9feb772 --- /dev/null +++ b/datachecker-common/pom.xml @@ -0,0 +1,74 @@ + + + 4.0.0 + + org.opengauss + openGauss-tools-datachecker-performance + 0.0.1 + ../pom.xml + + datachecker-common + 0.0.1 + datachecker-common + jar + datachecker-common + + 11 + 0.0.1 + + + + org.springframework.boot + spring-boot-starter-web + + + org.springdoc + springdoc-openapi-ui + + + net.openhft + zero-allocation-hashing + + + com.alibaba + fastjson + + + org.projectlombok + lombok + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + none + execute + + + org.projectlombok + lombok + + + + + + + repackage + + + + + + + + diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/Constants.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/Constants.java new file mode 100644 index 0000000..0eea78c --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/constant/Constants.java @@ -0,0 +1,12 @@ +package org.opengauss.datachecker.common.constant; + +/** + * 系统常量定义 + */ +public interface Constants { + String PRIMARY_DELIMITER = "_#_"; + + interface InitialCapacity { + int MAP = 0; + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/DataCheckParam.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/DataCheckParam.java new file mode 100644 index 0000000..0a0e494 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/DataCheckParam.java @@ -0,0 +1,52 @@ +package org.opengauss.datachecker.common.entry.check; + +import lombok.Getter; +import org.opengauss.datachecker.common.entry.extract.Topic; +import org.springframework.lang.NonNull; + +/** + * 数据校验线程参数 + * + * @author ：wangchao + * @date ：Created in 2022/6/10 + * @since ：11 + */ +@Getter +public class DataCheckParam { + + /** + * 构建桶容量参数 + */ + private final int bucketCapacity; + + /** + * 数据校验TOPIC对象 + */ + private final Topic topic; + /** + * 校验Topic 分区 + */ + private final int partitions; + /** + * 校验结果输出路径 + */ + private final String path; + + private final String schema; + + /** + * 校验参数构建器 + * + * @param bucketCapacity 构建桶容量参数 + * @param topic 数据校验TOPIC对象 + * @param partitions 校验Topic 分区 + * @param path 校验结果输出路径 + */ + public DataCheckParam(int bucketCapacity, @NonNull Topic topic, int partitions, @NonNull String path, String schema) { + this.bucketCapacity = bucketCapacity; + this.topic = topic; + this.partitions = partitions; + this.path = path; + this.schema = schema; + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/DifferencePair.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/DifferencePair.java new file mode 100644 index 0000000..0eaa475 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/DifferencePair.java @@ -0,0 +1,44 @@ +package org.opengauss.datachecker.common.entry.check; + +import lombok.EqualsAndHashCode; +import lombok.Getter; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/1 + * @since ：11 + */ +@Getter +@EqualsAndHashCode +public final class DifferencePair { + + private final L onlyOnLeft; + private final R onlyOnRight; + private final D differing; + + private DifferencePair(L onlyOnLeft, R onlyOnRight, D differing) { + this.onlyOnLeft = onlyOnLeft; + this.onlyOnRight = onlyOnRight; + this.differing = differing; + } + + /** + * Creates a new {@link DifferencePair} for the given elements. + * + * @param onlyOnLeft must not be {@literal null}. + * @param onlyOnRight must not be {@literal null}. + * @param differing must not be {@literal null}. + * @return + */ + public static DifferencePair of(L onlyOnLeft, R onlyOnRight, D differing) { + return new DifferencePair<>(onlyOnLeft, onlyOnRight, differing); + } + + /** + * @return differing : onlyOnLeft -> onlyOnRight + */ + @Override + public String toString() { + return String.format("%s : %s->%s", this.differing, this.onlyOnLeft, this.onlyOnRight); + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/Pair.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/Pair.java new file mode 100644 index 0000000..ba42658 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/check/Pair.java @@ -0,0 +1,61 @@ +package org.opengauss.datachecker.common.entry.check; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.springframework.lang.NonNull; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/1 + * @since ：11 + */ +@Getter +@EqualsAndHashCode +public final class Pair { + + private S source; + private T sink; + + private Pair(S source, T sink) { + this.source = source; + this.sink = sink; + } + + /** + * Creates a new {@link Pair} for the given elements. + * + * @param source must not be {@literal null}. + * @param sink must not be {@literal null}. + * @return + */ + public static Pair of(S source, T sink) { + return new Pair<>(source, sink); + } + + /** + * 修改pair + * + * @param pair + * @param sink + * @param + * @param + * @return + */ + public static Pair of(@NonNull Pair pair, T sink) { + pair.sink = sink; + return pair; + } + + public static Pair of(S source, @NonNull Pair pair) { + pair.source = source; + return pair; + } + + /** + * @return source->sink + */ + @Override + public String toString() { + return String.format("%s->%s", this.source, this.sink); + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/CheckBlackWhiteMode.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/CheckBlackWhiteMode.java new file mode 100644 index 0000000..a995306 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/CheckBlackWhiteMode.java @@ -0,0 +1,40 @@ +package org.opengauss.datachecker.common.entry.enums; + +import lombok.Getter; + +/** + * {@value API_DESCRIPTION } + * + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +@Getter +public enum CheckBlackWhiteMode implements IEnum { + /** + * 不开启黑白名单模式 + */ + NONE("NONE", "do not turn on black and white list mode"), + /** + * 黑名单校验 + */ + BLACK("BLACK", "blacklist verification mode"), + /** + * 白名单校验 + */ + WHITE("WHITE", "white list verification mode"); + + private final String code; + private final String description; + + CheckBlackWhiteMode(String code, String description) { + this.code = code; + this.description = description; + } + + public static final String API_DESCRIPTION = "black and white list verification mode [" + + " NONE-do not turn on black and white list mode," + + " BLACK-blacklist verification mode," + + " WHITE-white list verification mode" + + "]"; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/CheckMode.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/CheckMode.java new file mode 100644 index 0000000..d800801 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/CheckMode.java @@ -0,0 +1,32 @@ +package org.opengauss.datachecker.common.entry.enums; + +import lombok.Getter; + +/** + * 校验方式 + * + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +@Getter +public enum CheckMode implements IEnum { + /** + * 全量校验 + */ + FULL("FULL", "full check mode"), + /** + * 增量校验 + */ + INCREMENT("INCREMENT", "increment check mode"); + + private final String code; + private final String description; + + CheckMode(String code, String description) { + this.code = code; + this.description = description; + } + + public static final String API_DESCRIPTION = "CheckMode [FULL-full check mode,INCREMENT-increment check mode]"; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/ColumnKey.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/ColumnKey.java new file mode 100644 index 0000000..39bf174 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/ColumnKey.java @@ -0,0 +1,28 @@ +package org.opengauss.datachecker.common.entry.enums; + +import lombok.Getter; + + +@Getter +public enum ColumnKey implements IEnum { + /** + * 主键 + */ + PRI("PRI"), + /** + * UNI + */ + UNI("UNI"), + /** + * MUL + */ + MUL("MUL"); + + private final String code; + private String description; + + ColumnKey(String code) { + this.code = code; + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DML.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DML.java new file mode 100644 index 0000000..40d78bb --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DML.java @@ -0,0 +1,35 @@ +package org.opengauss.datachecker.common.entry.enums; + +import lombok.Getter; + +/** + * {@value API_DESCRIPTION} + * @author ：wangchao + * @date ：Created in 2022/6/12 + * @since ：11 + */ +@Getter +public enum DML implements IEnum { + /** + * Insert插入语句 + */ + INSERT("INSERT", "InsertStatement"), + /** + * Delete删除语句 + */ + DELETE("DELETE", "DeleteStatement"), + /** + * Replace修改语句 + */ + REPLACE("REPLACE", "ReplaceStatement"); + + private final String code; + private final String description; + + DML(String code, String description) { + this.code = code; + this.description = description; + } + + public static final String API_DESCRIPTION = "DML [INSERT-InsertStatement,DELETE-DeleteStatement,REPLACE-ReplaceStatement]"; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataBaseMeta.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataBaseMeta.java new file mode 100644 index 0000000..4e56607 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataBaseMeta.java @@ -0,0 +1,24 @@ +package org.opengauss.datachecker.common.entry.enums; + +import lombok.Getter; + +@Getter +public enum DataBaseMeta implements IEnum { + /** + * TableMetaData + */ + TABLE("TableMetaData", "TableMetaData"), + /** + * TablesColumnMetaData + */ + COLUMN("TablesColumnMetaData", "TablesColumnMetaData"); + + private final String code; + private final String description; + + DataBaseMeta(String code, String description) { + this.code = code; + this.description = description; + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataBaseType.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataBaseType.java new file mode 100644 index 0000000..2ba577b --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataBaseType.java @@ -0,0 +1,32 @@ +package org.opengauss.datachecker.common.entry.enums; + +import lombok.Getter; + +/** + * {@value API_DESCRIPTION} + */ +@Getter +public enum DataBaseType implements IEnum { + /** + * MySQL数据库类型 + */ + MS("MYSQL", "MYSQL"), + /** + * open gauss数据库 + */ + OG("OPENGAUSS", "OPENGAUSS"), + /** + * oracle数据库 + */ + O("ORACLE", "ORACLE"); + + private final String code; + private final String description; + + DataBaseType(String code, String description) { + this.code = code; + this.description = description; + } + + public static final String API_DESCRIPTION = "Database type [MS-MYSQL,OG-OPENGAUSS,O-ORACLE]"; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataSourceType.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataSourceType.java new file mode 100644 index 0000000..2860916 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/DataSourceType.java @@ -0,0 +1,23 @@ +package org.opengauss.datachecker.common.entry.enums; + +import lombok.Getter; + +@Getter +public enum DataSourceType implements IEnum { + /** + * 源端 + */ + Source("Source"), + /** + * 宿端 + */ + Sink("Sink"); + + private final String code; + private String description; + + DataSourceType(String code) { + this.code = code; + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/Endpoint.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/Endpoint.java new file mode 100644 index 0000000..1eca629 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/Endpoint.java @@ -0,0 +1,39 @@ +package org.opengauss.datachecker.common.entry.enums; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Getter; + +/** + * {@value API_DESCRIPTION} + * + * @author ：wangchao + * @date ：Created in 2022/5/25 + * @since ：11 + */ +@Schema(description = "data verification endpoint type") +@Getter +public enum Endpoint { + /** + * 源端 + */ + SOURCE(1, "SourceEndpoint"), + /** + * 宿端 + */ + SINK(2, "SinkEndpoint"), + /** + * 校验端 + */ + CHECK(3, "CheckEndpoint"); + + private final int code; + private final String description; + + Endpoint(int code, String description) { + this.code = code; + this.description = description; + } + + public static final String API_DESCRIPTION = "data verification endpoint type " + + "[SOURCE-1-SourceEndpoint,SINK-2-SinkEndpoint,CHECK-3-CheckEndpoint]"; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/IEnum.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/IEnum.java new file mode 100644 index 0000000..6ae625e --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/IEnum.java @@ -0,0 +1,17 @@ +package org.opengauss.datachecker.common.entry.enums; + +public interface IEnum { + /** + * 定义枚举code + * + * @return 返回枚举code + */ + String getCode(); + + /** + * 声明枚举描述 + * + * @return 返回枚举描述 + */ + String getDescription(); +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/ResultEnum.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/ResultEnum.java new file mode 100644 index 0000000..bc396ed --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/enums/ResultEnum.java @@ -0,0 +1,81 @@ +package org.opengauss.datachecker.common.entry.enums; + +import lombok.Getter; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/26 + * @since ：11 + */ +@Getter +public enum ResultEnum { + + //自定义系列 校验异常 + //校验服务异常： + CHECKING(1000, "verification service exception："), + //校验服务地址端口冲突 + CHECKING_ADDRESS_CONFLICT(1001, "verify service address port conflict."), + //校验服务Meta数据异常 + CHECK_META_DATA(1002, "verification service meta data exception."), + //校验服务-表数据差异过大，无法校验 + LARGE_DATA_DIFF(1003, "verification service - the table data difference is too large to be verified."), + //校验服务-默克尔树高度不一致 + MERKLE_TREE_DEPTH(1004, "verification service - height of Merkel tree is inconsistent."), + + //自定义系列 抽取异常 + //抽取服务异常 + EXTRACT(2000, "extraction service exception:"), + //创建KafkaTopic异常： + CREATE_TOPIC(2001, "create kafka topic exception:"), + //当前实例正在执行数据抽取服务，不能重新开启新的校验。 + PROCESS_MULTIPLE(2002, "The current instance is executing the data extraction service and cannot restart the new verification."), + //数据抽取服务，未找到待执行抽取任务 + TASK_NOT_FOUND(2003, "data extraction service, no extraction task to be executed found."), + //数据抽取服务，当前表对应元数据不存在 + TABLE_NOT_FOUND(2004, "data extraction service. The metadata corresponding to the current table does not exist."), + //Debezium配置错误 + DEBEZIUM_CONFIG_ERROR(2005, "debezium configuration error"), + + //自定义系列 抽取异常 + //Feign客户端异常 + FEIGN_CLIENT(3000, "feign client exception"), + //调度Feign客户端异常 + DISPATCH_CLIENT(3001, "scheduling feign client exception"), + + SUCCESS(200, "SUCCESS"), + SERVER_ERROR(400, "ERROR"), + //400系列 + //请求的数据格式不符 + BAD_REQUEST(400, "The requested data format does not match!"), + //登录凭证过期! + UNAUTHORIZED(401, "login certificate expired!"), + //抱歉，你无权限访问! + FORBIDDEN(403, "Sorry, you have no access!"), + //请求的资源找不到! + NOT_FOUND(404, "The requested resource cannot be found!"), + //参数丢失 + PARAM_MISSING(405, "Parameter missing"), + //参数类型不匹配 + PARAM_TYPE_MISMATCH(406, "Parameter type mismatch"), + //请求方法不支持 + HTTP_REQUEST_METHOD_NOT_SUPPORTED_ERROR(407, "request method is not supported"), + //非法参数异常 + SERVER_ERROR_PRARM(408, "illegal parameter exception"), + + //500系列 + //服务器内部错误! + INTERNAL_SERVER_ERROR(500, "server internal error!"), + //服务器正忙，请稍后再试! + SERVICE_UNAVAILABLE(503, "the server is busy, please try again later!"), + + //未知异常 + UNKNOWN(7000, "Unknown exception!"); + private final int code; + private final String description; + + ResultEnum(int code, String description) { + this.code = code; + this.description = description; + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/CheckDiffResult.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/CheckDiffResult.java new file mode 100644 index 0000000..1e1e7ef --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/CheckDiffResult.java @@ -0,0 +1,44 @@ +//package org.opengauss.datachecker.common.entry.extract; +// +//import lombok.Data; +//import lombok.experimental.Accessors; +//import lombok.experimental.SuperBuilder; +// +//import java.time.LocalDateTime; +//import java.util.List; +//import java.util.Set; +// +///** +// * @author ：wangchao +// * @date ：Created in 2022/6/18 +// * @since ：11 +// */ +//@Data +//@Accessors(chain = true) +//public class CheckDiffResult { +// private String table; +// private int partitions; +// private String topic; +// private LocalDateTime createTime; +// +// private Set keyUpdateSet; +// private Set keyInsertSet; +// private Set keyDeleteSet; +// +// private List repairUpdate; +// private List repairInsert; +// private List repairDelete; +// +// public CheckDiffResult(final CheckDiffResultBuilder b) { +// this.table = b.table; +// this.partitions = b.partitions; +// this.topic = b.topic; +// this.createTime = b.createTime; +// this.keyUpdateSet = b.keyUpdateSet; +// this.keyInsertSet = b.keyInsertSet; +// this.keyDeleteSet = b.keyDeleteSet; +// this.repairUpdate = b.repairUpdate; +// this.repairInsert = b.repairInsert; +// this.repairDelete = b.repairDelete; +// } +//} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ColumnsMetaData.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ColumnsMetaData.java new file mode 100644 index 0000000..5c6aa9d --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ColumnsMetaData.java @@ -0,0 +1,40 @@ +package org.opengauss.datachecker.common.entry.extract; + +import lombok.Data; +import lombok.ToString; +import lombok.experimental.Accessors; +import org.opengauss.datachecker.common.entry.enums.ColumnKey; + +/** + * 表元数据信息 + */ +@Data +@Accessors(chain = true) +@ToString +public class ColumnsMetaData { + /** + * 表名 + */ + private String tableName; + /** + * 主键列名称 + */ + private String columnName; + /** + * 主键列数据类型 + */ + private String columnType; + /** + * 主键列数据类型 + */ + private String dataType; + /** + * 主键表序号 + */ + private int ordinalPosition; + /** + * 主键 + */ + private ColumnKey columnKey; +} + diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractIncrementTask.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractIncrementTask.java new file mode 100644 index 0000000..ee30b14 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractIncrementTask.java @@ -0,0 +1,33 @@ +package org.opengauss.datachecker.common.entry.extract; + +import lombok.Data; +import lombok.ToString; +import lombok.experimental.Accessors; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/14 + * @since ：11 + */ +@ToString +@Data +@Accessors(chain = true) +public class ExtractIncrementTask { + /** + * 表名称 + */ + private String tableName; + /** + * 当前抽取端点 schema + */ + private String schema; + /** + * 任务名称 + */ + private String taskName; + + /** + * 数据变更日志 + */ + private SourceDataLog sourceDataLog; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractTask.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractTask.java new file mode 100644 index 0000000..24152f9 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/ExtractTask.java @@ -0,0 +1,44 @@ +package org.opengauss.datachecker.common.entry.extract; + +import lombok.Data; +import lombok.ToString; +import lombok.experimental.Accessors; + +@ToString +@Data +@Accessors(chain = true) +public class ExtractTask { + /** + * 任务名称 + */ + private String taskName; + /** + * 表名称 + */ + private String tableName; + + /** + * 任务分拆总数：1 表示未分拆，大于1则表示分拆为divisionsTotalNumber个任务 + */ + private int divisionsTotalNumber; + /** + * 当前表，拆分任务序列 + */ + private int divisionsOrdinal; + /** + * 任务执行起始位置 + */ + private long start; + /** + * 任务执行偏移量 + */ + private long offset; + /** + * 表元数据信息 + */ + private TableMetadata tableMetadata; + + public boolean isDivisions() { + return divisionsTotalNumber > 1; + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/PrimaryMeta.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/PrimaryMeta.java new file mode 100644 index 0000000..45d13fc --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/PrimaryMeta.java @@ -0,0 +1,21 @@ +package org.opengauss.datachecker.common.entry.extract; + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors +public class PrimaryMeta { + /** + * 主键列名称 + */ + private String columnName; + /** + * 主键列数据类型 + */ + private String columnType; + /** + * 主键表序号 + */ + private int ordinalPosition; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/RowDataHash.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/RowDataHash.java new file mode 100644 index 0000000..21db8f9 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/RowDataHash.java @@ -0,0 +1,25 @@ +package org.opengauss.datachecker.common.entry.extract; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; + +@Data +@EqualsAndHashCode +@Accessors(chain = true) +public class RowDataHash { + + /** + * 主键为数字类型则 转字符串，表主键为联合主键，则当前属性为表主键联合字段对应值 拼接字符串 以下划线拼接 + */ + private String primaryKey; + + /** + * 主键对应值的哈希值 + */ + private long primaryKeyHash; + /** + * 当前记录的总体哈希值 + */ + private long rowHash; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SourceDataLog.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SourceDataLog.java new file mode 100644 index 0000000..206e4eb --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/SourceDataLog.java @@ -0,0 +1,42 @@ +package org.opengauss.datachecker.common.entry.extract; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.experimental.Accessors; +import org.opengauss.datachecker.common.constant.Constants; + +import java.util.List; + +/** + * 源端数据变更日志 + * + * @author ：wangchao + * @date ：Created in 2022/6/14 + * @since ：11 + */ +@Schema(description = "源端数据变更日志") +@Data +@Accessors(chain = true) +public class SourceDataLog { + + private static final String PRIMARY_DELIMITER = Constants.PRIMARY_DELIMITER; + /** + * 数据变更日志 对应表名称 + */ + @Schema(name = "tableName", description = "表名称") + private String tableName; + + /** + * 当前表的主键字段名称列表 + */ + @Schema(name = "compositePrimarys", description = "当前表的主键字段名称列表") + private List compositePrimarys; + + /** + * 相同数据操作类型{@code operateCategory}的数据变更的主键值列表

+ * 单主键表 ：主键值直接添加进{@code compositePrimarysValues}集合。

+ * 复合主键：对主键值进行组装，根据{@code compositePrimarys}记录的主键字段顺序，进行拼接。链接符{@value PRIMARY_DELIMITER} + */ + @Schema(name = "compositePrimaryValues", description = "相同数据操作类型{@code operateCategory}的数据变更的主键值列表") + private List compositePrimaryValues; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadata.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadata.java new file mode 100644 index 0000000..17d20ab --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadata.java @@ -0,0 +1,39 @@ +package org.opengauss.datachecker.common.entry.extract; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.ToString; +import lombok.experimental.Accessors; + +import java.util.List; + +/** + * 表元数据信息 + */ +@Schema(name = "表元数据信息") +@Data +@Accessors(chain = true) +@ToString +public class TableMetadata { + + /** + * 表名 + */ + private String tableName; + /** + * 表数据总量 + */ + private long tableRows; + + /** + * 主键列属性 + */ + private List primaryMetas; + + /** + * 表列属性 + */ + private List columnsMetas; + +} + diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadataHash.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadataHash.java new file mode 100644 index 0000000..9ce461a --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/TableMetadataHash.java @@ -0,0 +1,21 @@ +package org.opengauss.datachecker.common.entry.extract; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; + +@Data +@EqualsAndHashCode +@Accessors(chain = true) +public class TableMetadataHash { + + /** + * 表名 + */ + private String tableName; + + /** + * 当前记录的总体哈希值 + */ + private long tableHash; +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/Topic.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/Topic.java new file mode 100644 index 0000000..8d89d2d --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/entry/extract/Topic.java @@ -0,0 +1,24 @@ +package org.opengauss.datachecker.common.entry.extract; + +import lombok.Data; +import lombok.ToString; +import lombok.experimental.Accessors; + +@ToString +@Data +@Accessors(chain = true) +public class Topic { + /** + * 表名称 + */ + private String tableName; + /** + * 当前表，对应的Topic名称 + */ + private String topicName; + /** + * 当前表存在在Kafka Topic中的数据的分区总数 + */ + private int partitions; + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckMetaDataException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckMetaDataException.java new file mode 100644 index 0000000..4796e8b --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckMetaDataException.java @@ -0,0 +1,16 @@ +package org.opengauss.datachecker.common.exception; + +/** + * 校验服务 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class CheckMetaDataException extends CheckingException { + + public CheckMetaDataException(String message) { + super(message); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingAddressConflictException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingAddressConflictException.java new file mode 100644 index 0000000..040aacc --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingAddressConflictException.java @@ -0,0 +1,16 @@ +package org.opengauss.datachecker.common.exception; + +/** + * 校验服务 配置源端和宿端地址进行约束性检查：源端和宿端地址不能重复 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class CheckingAddressConflictException extends CheckingException { + + public CheckingAddressConflictException(String message) { + super(message); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingException.java new file mode 100644 index 0000000..15446a1 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingException.java @@ -0,0 +1,20 @@ +package org.opengauss.datachecker.common.exception; + +import java.util.Objects; + +public class CheckingException extends RuntimeException { + + private final String msg; + + public CheckingException(String message) { + this.msg = message; + } + @Override + public String getMessage() { + String message = super.getMessage(); + if (Objects.isNull(message)) { + return this.msg; + } + return this.msg + message; + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingPollingException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingPollingException.java new file mode 100644 index 0000000..54bc8a2 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CheckingPollingException.java @@ -0,0 +1,16 @@ +package org.opengauss.datachecker.common.exception; + +/** + * 校验服务 校验轮询异常 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class CheckingPollingException extends CheckingException { + + public CheckingPollingException(String message) { + super(message); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CreateTopicException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CreateTopicException.java new file mode 100644 index 0000000..a8fc6f2 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/CreateTopicException.java @@ -0,0 +1,32 @@ +/* + * + * * Copyright 2019-2020 the original author or authors. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.opengauss.datachecker.common.exception; + +@SuppressWarnings("serial") +public class CreateTopicException extends ExtractException { + private final String msg; + + public CreateTopicException(String message) { + this.msg = message; + } + @Override + public String getMessage() { + return this.msg + "_" + super.getMessage(); + } +} \ No newline at end of file diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/DispatchClientException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/DispatchClientException.java new file mode 100644 index 0000000..0c0d9fe --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/DispatchClientException.java @@ -0,0 +1,25 @@ +package org.opengauss.datachecker.common.exception; + +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.springframework.lang.NonNull; + +/** + * 调度客户端异常 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class DispatchClientException extends FeignClientException { + + /** + * 调度客户端异常 + * + * @param endpoint 端点 + * @param message 异常信息 + */ + public DispatchClientException(@NonNull Endpoint endpoint, String message) { + super(endpoint, message); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/ExtractException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/ExtractException.java new file mode 100644 index 0000000..9f29c4f --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/ExtractException.java @@ -0,0 +1,17 @@ +package org.opengauss.datachecker.common.exception; + +import lombok.Getter; + +@Getter +public class ExtractException extends RuntimeException { + + //数据抽取服务异常 + private String message = "Data extraction service exception"; + + public ExtractException(String message) { + this.message = message; + } + + public ExtractException() { + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/FeignClientException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/FeignClientException.java new file mode 100644 index 0000000..ed6847b --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/FeignClientException.java @@ -0,0 +1,19 @@ +package org.opengauss.datachecker.common.exception; + +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.springframework.lang.NonNull; + +/** + * 工具FeignClient 调用异常 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class FeignClientException extends RuntimeException { + + public FeignClientException(@NonNull Endpoint endpoint, String message) { + super(endpoint.getDescription() + " " + message); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/GlobalCommonExceptionHandler.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/GlobalCommonExceptionHandler.java new file mode 100644 index 0000000..cbcd753 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/GlobalCommonExceptionHandler.java @@ -0,0 +1,80 @@ +package org.opengauss.datachecker.common.exception; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.entry.enums.ResultEnum; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.web.HttpRequestMethodNotSupportedException; +import org.springframework.web.bind.MissingServletRequestParameterException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; + + +import javax.servlet.http.HttpServletRequest; + +@Slf4j +public class GlobalCommonExceptionHandler { + + /** + * 缺少必要的参数 + */ + @ExceptionHandler(value = MissingServletRequestParameterException.class) + public Result missingParameterHandler(HttpServletRequest request, MissingServletRequestParameterException e) { + this.logError(request, e); + return Result.fail(ResultEnum.PARAM_MISSING); + } + + /** + * 参数类型不匹配 + */ + @ExceptionHandler(value = MethodArgumentTypeMismatchException.class) + public Result methodArgumentTypeMismatchException(HttpServletRequest request, MethodArgumentTypeMismatchException e) { + this.logError(request, e); + return Result.fail(ResultEnum.PARAM_TYPE_MISMATCH); + } + + /** + * 不支持的请求方法 + */ + @ExceptionHandler(value = HttpRequestMethodNotSupportedException.class) + public Result httpRequestMethodNotSupportedException(HttpServletRequest request, HttpRequestMethodNotSupportedException e) { + this.logError(request, e); + return Result.fail(ResultEnum.HTTP_REQUEST_METHOD_NOT_SUPPORTED_ERROR); + } + + /** + * 参数错误 + */ + @ExceptionHandler(value = IllegalArgumentException.class) + public Result illegalArgumentException(HttpServletRequest request, IllegalArgumentException e) { + this.logError(request, e); + return Result.fail(ResultEnum.SERVER_ERROR_PRARM); + } + + @ExceptionHandler(value = FeignClientException.class) + public Result feignClientException(HttpServletRequest request, FeignClientException e) { + this.logError(request, e); + return Result.fail(ResultEnum.FEIGN_CLIENT); + } + + @ExceptionHandler(value = DispatchClientException.class) + public Result dispatchClientException(HttpServletRequest request, DispatchClientException e) { + this.logError(request, e); + return Result.fail(ResultEnum.DISPATCH_CLIENT); + } + + /** + * 其他异常统一处理 + */ + @ExceptionHandler(value = Exception.class) + public Result exception(HttpServletRequest request, Exception e) { + this.logError(request, e); + return Result.fail(ResultEnum.SERVER_ERROR); + } + + /** + * 记录错误日志 + */ + protected void logError(HttpServletRequest request, Exception e) { + log.error("path:{}, queryParam:{}, errorMessage:{}", request.getRequestURI(), request.getQueryString(), e.getMessage(), e); + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/LargeDataDiffException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/LargeDataDiffException.java new file mode 100644 index 0000000..7971a66 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/LargeDataDiffException.java @@ -0,0 +1,16 @@ +package org.opengauss.datachecker.common.exception; + +/** + * 校验服务 数据量产生过大差异，无法进行校验。 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class LargeDataDiffException extends CheckingException { + + public LargeDataDiffException(String message) { + super(message); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/MerkleTreeDepthException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/MerkleTreeDepthException.java new file mode 100644 index 0000000..10df9ef --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/MerkleTreeDepthException.java @@ -0,0 +1,16 @@ +package org.opengauss.datachecker.common.exception; + +/** + * 校验服务 数据量产生过大差异，导致构建默克尔树高度不一致，无法进行校验。 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class MerkleTreeDepthException extends LargeDataDiffException { + + public MerkleTreeDepthException(String message) { + super(message); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/ProcessMultipleException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/ProcessMultipleException.java new file mode 100644 index 0000000..e5fc7f9 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/ProcessMultipleException.java @@ -0,0 +1,16 @@ +package org.opengauss.datachecker.common.exception; + +/** + * 当前实例正在执行数据抽取服务，不能重新开启新的校验。 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class ProcessMultipleException extends ExtractException { + + public ProcessMultipleException(String message) { + super(message); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/TableNotExistException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/TableNotExistException.java new file mode 100644 index 0000000..6b4be3e --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/TableNotExistException.java @@ -0,0 +1,19 @@ +package org.opengauss.datachecker.common.exception; + + +/** + * 数据抽取服务，表元数据信息不存在 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class TableNotExistException extends ExtractException { + private static final String ERROR_MESSAGE = "table of Meatedata [%s] is not exist!"; + + + public TableNotExistException(String tableName) { + super(String.format(ERROR_MESSAGE, tableName)); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/TaskNotFoundException.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/TaskNotFoundException.java new file mode 100644 index 0000000..aca470c --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/exception/TaskNotFoundException.java @@ -0,0 +1,24 @@ +package org.opengauss.datachecker.common.exception; + +import org.opengauss.datachecker.common.entry.enums.Endpoint; + +/** + * 数据抽取服务，未找到待执行抽取任务 + * + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class TaskNotFoundException extends ExtractException { + private static final String ERROR_MESSAGE = "task %s is not found,please checking something error!"; + private static final String ERROR_ENDPOINT_MESSAGE = "endpoint [%s] and process[%s] task is empty!"; + + public TaskNotFoundException(Endpoint endpoint, long process) { + super(String.format(ERROR_ENDPOINT_MESSAGE, endpoint.getDescription(), process)); + } + + public TaskNotFoundException(String taskName) { + super(String.format(ERROR_MESSAGE, taskName)); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ByteUtil.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ByteUtil.java new file mode 100644 index 0000000..f71a1ea --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ByteUtil.java @@ -0,0 +1,42 @@ +package org.opengauss.datachecker.common.util; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +public class ByteUtil { + + /** + * 比较两个字节数组是否一致 + * + * @param byte1 字节数组 + * @param byte2 字节数组 + * @return true|false + */ + public static boolean isEqual(byte[] byte1, byte[] byte2) { + if (byte1 == null || byte2 == null || byte1.length != byte2.length) { + return false; + } + return HashUtil.hashBytes(byte1) == HashUtil.hashBytes(byte2); + } + + /** + * 将long型数字转化为byte字节数组 + * + * @param value long型数组 + * @return 字节数组 + */ + public static byte[] toBytes(long value) { + return new byte[]{ + (byte) (value >> 56), + (byte) (value >> 48), + (byte) (value >> 40), + (byte) (value >> 32), + (byte) (value >> 24), + (byte) (value >> 16), + (byte) (value >> 8), + (byte) value + }; + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/EnumUtil.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/EnumUtil.java new file mode 100644 index 0000000..b84e827 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/EnumUtil.java @@ -0,0 +1,69 @@ +package org.opengauss.datachecker.common.util; + +import org.opengauss.datachecker.common.entry.enums.IEnum; + +public class EnumUtil { + + /** + * 获取枚举 + * + * @param clazz + * @param code + * @return + */ + public static T valueOfIgnoreCase(Class clazz, String code) { + return valueOf(clazz, code, true); + } + + /** + * 获取枚举,区分大小写 + * + * @param clazz + * @param code + * @param isIgnore + * @return + */ + public static T valueOf(Class clazz, String code, boolean isIgnore) { + + //得到values + T[] enums = values(clazz); + + if (enums == null || enums.length == 0) { + return null; + } + + for (T t : enums) { + if (isIgnore && t.getCode().equalsIgnoreCase(code)) { + return t; + } else if (t.getCode().equals(code)) { + return t; + } + } + return null; + } + /** + * 获取枚举,区分大小写 + * + * @param clazz + * @param code + * @return + */ + public static T valueOf(Class clazz, String code) { + return valueOf(clazz, code, false); + } + + /** + * 获取枚举集合 + * + * @param clazz + * @return + */ + public static T[] values(Class clazz) { + if (!clazz.isEnum()) { + throw new IllegalArgumentException("Class[" + clazz + "]不是枚举类型"); + } + //得到values + return clazz.getEnumConstants(); + } + +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/FileUtils.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/FileUtils.java new file mode 100644 index 0000000..fde4137 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/FileUtils.java @@ -0,0 +1,57 @@ +package org.opengauss.datachecker.common.util; + +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Set; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +@Slf4j +public class FileUtils { + + public static void createDirectories(String path) { + File file = new File(path); + if (!file.exists()) { + try { + Files.createDirectories(Paths.get(path)); + } catch (IOException e) { + log.error("createDirectories error:", e); + } + } + } + + public static void writeAppendFile(String filename, List content) { + try { + Files.write(Paths.get(filename), content, StandardOpenOption.APPEND, StandardOpenOption.CREATE); + } catch (IOException e) { + log.error("file write error:", e); + } + } + + + public static void writeAppendFile(String filename, Set content) { + try { + Files.write(Paths.get(filename), content, StandardOpenOption.APPEND, StandardOpenOption.CREATE); + } catch (IOException e) { + log.error("file write error:", e); + } + } + + public static void writeAppendFile(String filename, String content) { + try { + Files.write(Paths.get(filename), content.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND, StandardOpenOption.CREATE); + } catch (IOException e) { + log.error("file write error:", e); + } + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/HashUtil.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/HashUtil.java new file mode 100644 index 0000000..bfe9892 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/HashUtil.java @@ -0,0 +1,49 @@ +package org.opengauss.datachecker.common.util; + +import net.openhft.hashing.LongHashFunction; +import org.springframework.lang.NonNull; + +import java.nio.charset.Charset; + + +/** + * @author ：wangchao + * @date ：Created in 2022/5/24 + * @since ：11 + */ +public class HashUtil { + + /** + * 哈希算法 + */ + private static final LongHashFunction XX_3_HASH = LongHashFunction.xx3(); + + /** + * 使用xx3哈希算法 对字符串进行哈希计算 + * + * @param input 字符串 + * @return 哈希值 + */ + public static long hashChars(@NonNull String input) { + return XX_3_HASH.hashChars(input); + } + /** + * 使用xx3哈希算法 对字节数组进行哈希计算 + * + * @param input 字节数组 + * @return 哈希值 + */ + public static long hashBytes(@NonNull byte[] input) { + return XX_3_HASH.hashBytes(input); + } + + /** + * 使用xx3哈希算法 对字符串进行哈希计算 + * + * @param input 字符串 + * @return 哈希值 + */ + public static long hashBytes(@NonNull String input) { + return XX_3_HASH.hashBytes(input.getBytes(Charset.defaultCharset())); + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/IdWorker.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/IdWorker.java new file mode 100644 index 0000000..e030f47 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/IdWorker.java @@ -0,0 +1,224 @@ +package org.opengauss.datachecker.common.util; + +import java.lang.management.ManagementFactory; +import java.net.InetAddress; +import java.net.NetworkInterface; + +/** + *

描述：自增长ID Snowflake 雪花算法 Java实现

+ * 核心代码为其IdWorker这个类实现，其原理结构如下，我分别用一个0表示一位，用—分割开部分的作用： + * 1||0---0000000000 0000000000 0000000000 0000000000 0 --- 00000 ---00000 ---000000000000 + * 在上面的字符串中，第一位为未使用（实际上也可作为long的符号位），接下来的41位为毫秒级时间， + * 然后5位data center标识位，5位机器ID（并不算标识符，实际是为线程标识）， + * 然后12位该毫秒内的当前毫秒内的计数，加起来刚好64位，为一个Long型。 + * 这样的好处是，整体上按照时间自增排序，并且整个分布式系统内不会产生ID碰撞（由datacenter和机器ID作区分）， + * 并且效率较高，经测试，snowflake每秒能够产生26万ID左右，完全满足需要。 + *

+ * 64位ID (42(毫秒)+5(机器ID)+5(业务编码)+12(重复累加)) + */ +public class IdWorker { + /** + * 时间起始标记点，作为基准，一般取系统的最近时间（一旦确定不能变动） + */ + private final static long BENCHMARK = 1653824897654L; + /** + * 机器标识位数 + */ + private final static long WORKER_ID_BITS = 5L; + /** + * 数据中心标识位数 + */ + private final static long DATACENTER_ID_BITS = 5L; + /** + * 机器ID最大值 + */ + private final static long MAX_WORKER_ID = ~(-1L << WORKER_ID_BITS); + /** + * 数据中心ID最大值 + */ + private final static long MAX_DATA_CENTER_ID = ~(-1L << DATACENTER_ID_BITS); + /** + * 毫秒内自增位 + */ + private final static long SEQUENCE_BITS = 12L; + /** + * 机器ID偏左移12位 + */ + private final static long WORKER_ID_SHIFT = SEQUENCE_BITS; + /** + * 数据中心ID左移17位 + */ + private final static long DATA_CENTER_ID_SHIFT = SEQUENCE_BITS + WORKER_ID_BITS; + /** + * 时间毫秒左移22位 + */ + private final static long TIMESTAMP_LEFT_SHIFT = SEQUENCE_BITS + WORKER_ID_BITS + DATACENTER_ID_BITS; + + private final static long SEQUENCE_MASK = ~(-1L << SEQUENCE_BITS); + /** + * 上次生产id时间戳 + */ + private static long lastTimestamp = -1L; + /** + * 0，并发控制 + */ + private volatile long sequence = 0L; + + private final long workerId; + /** + * 数据标识id部分 + */ + private final long dataCenterId; + + private static final Object LOCK = new Object(); + + private static IdWorker idWorker; + + public IdWorker() { + this.dataCenterId = getDataCenterId(MAX_DATA_CENTER_ID); + this.workerId = getMaxWorkerId(dataCenterId, MAX_WORKER_ID); + } + + /** + * @param workerId * 工作机器ID + * @param dataCenterId * 序列号 + */ + public IdWorker(long workerId, long dataCenterId) { + if (workerId > MAX_WORKER_ID || workerId < 0) { + throw new IllegalArgumentException(String.format("worker Id can't be greater than %d or less than 0", MAX_WORKER_ID)); + } + if (dataCenterId > MAX_DATA_CENTER_ID || dataCenterId < 0) { + throw new IllegalArgumentException(String.format("datacenter Id can't be greater than %d or less than 0", MAX_DATA_CENTER_ID)); + } + this.workerId = workerId; + this.dataCenterId = dataCenterId; + } + + public static IdWorker getInstance() { + if (idWorker != null) { + return idWorker; + } else { + synchronized (LOCK) { + if (idWorker == null) { + idWorker = new IdWorker(); + } + } + } + return idWorker; + } + + /** + * 获取ID自增序列 + * + * @return 返回下一个ID + */ + public static long nextId() { + return getInstance().next(); + } + + /** + * 36进制字符串ID + * + * @return 36进制字符串ID + */ + public static String nextId36() { + return Long.toString(getInstance().next(), Character.MAX_RADIX).toUpperCase(); + } + + /** + * 获取带有指定前缀的自增序列 + * + * @param prefix 自增序列前缀 + * @return 带指定前缀的下一个字符串ID + */ + public static String nextId(String prefix) { + return prefix + getInstance().next(); + } + + /** + * 获取下一个ID + * + * @return 返回下一个ID + */ + private synchronized long next() { + long timestamp = timeGen(); + if (timestamp < lastTimestamp) { + throw new RuntimeException(String.format("Clock moved backwards. Refusing to generate id for %d milliseconds", lastTimestamp - timestamp)); + } + + if (lastTimestamp == timestamp) { + // 当前毫秒内，则+1 + sequence = (sequence + 1) & SEQUENCE_MASK; + if (sequence == 0) { + // 当前毫秒内计数满了，则等待下一秒 + timestamp = tilNextMillis(lastTimestamp); + } + } else { + sequence = 0L; + } + lastTimestamp = timestamp; + // ID偏移组合生成最终的ID，并返回ID + long nextGenId = ((timestamp - BENCHMARK) << TIMESTAMP_LEFT_SHIFT) + | (dataCenterId << DATA_CENTER_ID_SHIFT) + | (workerId << WORKER_ID_SHIFT) | sequence; + + return nextGenId; + } + + private long tilNextMillis(final long lastTimestamp) { + long timestamp = this.timeGen(); + while (timestamp <= lastTimestamp) { + timestamp = this.timeGen(); + } + return timestamp; + } + + private long timeGen() { + return System.currentTimeMillis(); + } + + /** + *

+ * 获取 maxWorkerId + *

+ */ + protected static long getMaxWorkerId(long datacenterId, long maxWorkerId) { + StringBuffer mpid = new StringBuffer(); + mpid.append(datacenterId); + String name = ManagementFactory.getRuntimeMXBean().getName(); + if (!name.isEmpty()) { + /* + * GET jvmPid + */ + mpid.append(name.split("@")[0]); + } + /* + * MAC + PID 的 hashcode 获取16个低位 + */ + return (mpid.toString().hashCode() & 0xffff) % (maxWorkerId + 1); + } + + /** + *

+ * 数据标识id部分 + *

+ */ + protected static long getDataCenterId(long maxDataCenterId) { + long id = 0L; + try { + InetAddress ip = InetAddress.getLocalHost(); + NetworkInterface network = NetworkInterface.getByInetAddress(ip); + if (network == null) { + id = 1L; + } else { + byte[] mac = network.getHardwareAddress(); + id = ((0x000000FF & (long) mac[mac.length - 1]) + | (0x0000FF00 & (((long) mac[mac.length - 2]) << 8))) >> 6; + id = id % (maxDataCenterId + 1); + } + } catch (Exception e) { + System.out.println(" getDataCenterId: " + e.getMessage()); + } + return id; + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/JsonObjectUtil.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/JsonObjectUtil.java new file mode 100644 index 0000000..e8fe130 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/JsonObjectUtil.java @@ -0,0 +1,29 @@ +package org.opengauss.datachecker.common.util; + +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.serializer.SerializerFeature; +import lombok.extern.slf4j.Slf4j; + + +/** + * @author ：wangchao + * @date ：Created in 2022/5/23 + * @since ：11 + */ +@Slf4j +public class JsonObjectUtil { + + /** + * 对象格式化为JSON字符串，格式化根据属性进行自动换行

+ * {@code SerializerFeature.PrettyFormat}

+ * {@code SerializerFeature.WriteMapNullValue} 空指针格式化

+ * {@code SerializerFeature.WriteDateUseDateFormat} 日期格式化

+ * + * @param object 格式化对象 + * @return 格式化字符串 + */ + public static String format(Object object) { + return JSONObject.toJSONString(object, SerializerFeature.PrettyFormat, SerializerFeature.WriteMapNullValue, + SerializerFeature.WriteDateUseDateFormat); + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/SpringUtil.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/SpringUtil.java new file mode 100644 index 0000000..1e0501b --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/SpringUtil.java @@ -0,0 +1,59 @@ +package org.opengauss.datachecker.common.util; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +@Component +public class SpringUtil implements ApplicationContextAware { + /** + * 上下文对象实例 + */ + private static ApplicationContext applicationContext; + + @Autowired + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + SpringUtil.applicationContext = applicationContext; + } + + /** + * 获取applicationContext + * @return + */ + public static ApplicationContext getApplicationContext() { + return applicationContext; + } + + /** + * 通过name获取 Bean. + * @param name + * @return + */ + public static Object getBean(String name){ + return getApplicationContext().getBean(name); + } + + /** + * 通过class获取Bean. + * @param clazz + * @param + * @return + */ + public static T getBean(Class clazz){ + return getApplicationContext().getBean(clazz); + } + + /** + * 通过name,以及Clazz返回指定的Bean + * @param name + * @param clazz + * @param + * @return + */ + public static T getBean(String name,Class clazz){ + return getApplicationContext().getBean(name, clazz); + } +} \ No newline at end of file diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ThreadUtil.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ThreadUtil.java new file mode 100644 index 0000000..1428ab1 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/util/ThreadUtil.java @@ -0,0 +1,37 @@ +package org.opengauss.datachecker.common.util; + +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/31 + * @since ：11 + */ +@Slf4j +public class ThreadUtil { + /** + * 线程休眠 + * + * @param millisTime 休眠时间毫秒 + */ + public static void sleep(int millisTime) { + try { + Thread.sleep(millisTime); + } catch (InterruptedException ie) { + log.error("thread sleep interrupted exception "); + } + } + + public static ThreadPoolExecutor newSingleThreadExecutor() { + return new ThreadPoolExecutor(1, 1, 60L, TimeUnit.SECONDS, + new LinkedBlockingDeque<>(100), + Executors.defaultThreadFactory(), + new ThreadPoolExecutor.DiscardOldestPolicy()); + + } +} diff --git a/datachecker-common/src/main/java/org/opengauss/datachecker/common/web/Result.java b/datachecker-common/src/main/java/org/opengauss/datachecker/common/web/Result.java new file mode 100644 index 0000000..d42d656 --- /dev/null +++ b/datachecker-common/src/main/java/org/opengauss/datachecker/common/web/Result.java @@ -0,0 +1,70 @@ +package org.opengauss.datachecker.common.web; + +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.opengauss.datachecker.common.entry.enums.ResultEnum; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/26 + * @since ：11 + */ +@Tag(name = "API 接口消息返回结果封装类") +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Result { + + @Schema(name = "code", description = "消息响应码") + private int code; + + @Schema(name = "message", description = "消息内容") + private String message; + + @Schema(name = "data", description = "接口返回数据") + private T data; + + + public static Result success() { + return new Result<>(ResultEnum.SUCCESS.getCode(), ResultEnum.SUCCESS.getDescription(), null); + } + + public static Result of(T data) { + return new Result<>(ResultEnum.SUCCESS.getCode(), ResultEnum.SUCCESS.getDescription(), data); + } + + + public static Result of(T data, int code, String message) { + return new Result<>(code, message, data); + } + + public static Result fail(ResultEnum resultEnum) { + return new Result(resultEnum.getCode(), resultEnum.getDescription(), null); + } + + public static Result fail(ResultEnum resultEnum, String message) { + return new Result(resultEnum.getCode(), resultEnum.getDescription() + " " + message, null); + } + + public boolean isSuccess() { + return code == ResultEnum.SUCCESS.getCode(); + } + + public static Result success(T data) { + Result result = new Result<>(); + result.setCode(ResultEnum.SUCCESS.getCode()); + result.setMessage(ResultEnum.SUCCESS.getDescription()); + result.setData(data); + return result; + } + + public static Result error(String message) { + Result result = new Result<>(); + result.setCode(ResultEnum.SERVER_ERROR.getCode()); + result.setMessage(message); + return result; + } +} diff --git a/datachecker-common/src/main/resources/application.properties b/datachecker-common/src/main/resources/application.properties new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/datachecker-common/src/main/resources/application.properties @@ -0,0 +1 @@ + diff --git a/datachecker-common/src/test/java/org/opengauss/datachecker/common/DatacheckerCommonApplicationTests.java b/datachecker-common/src/test/java/org/opengauss/datachecker/common/DatacheckerCommonApplicationTests.java new file mode 100644 index 0000000..d69b4bd --- /dev/null +++ b/datachecker-common/src/test/java/org/opengauss/datachecker/common/DatacheckerCommonApplicationTests.java @@ -0,0 +1,13 @@ +package org.opengauss.datachecker.common; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class DatacheckerCommonApplicationTests { + + @Test + void contextLoads() { + } + +} diff --git a/datachecker-common/src/test/java/org/opengauss/datachecker/common/util/HashUtilTest.java b/datachecker-common/src/test/java/org/opengauss/datachecker/common/util/HashUtilTest.java new file mode 100644 index 0000000..e76fb2c --- /dev/null +++ b/datachecker-common/src/test/java/org/opengauss/datachecker/common/util/HashUtilTest.java @@ -0,0 +1,91 @@ +package org.opengauss.datachecker.common.util; + +import org.junit.jupiter.api.Test; + +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; + + +class HashUtilTest { + + @Test + void testHashBytes1() { + int mod = 600; + AtomicInteger min = new AtomicInteger(); + AtomicInteger max = new AtomicInteger(); + int[] resultCount = new int[mod * 2]; + IntStream.range(1, 100000).forEach(idx -> { + long x = HashUtil.hashBytes(UUID.randomUUID().toString().getBytes()); + int xmod = (int) (x % mod + mod); + max.set(Math.max(xmod, max.get())); + min.set(Math.min(xmod, min.get())); + resultCount[xmod]++; + //System.out.println(x + " " + xmod); + }); + System.out.println("min=" + min.get() + " max=" + max.get()); + IntStream.range(0, mod * 2).forEach(idx -> { + System.out.println("idx=" + idx + " " + resultCount[idx]); + }); + + } + + @Test + void testHashBytes2() { + System.out.println("bucket average capacity " + 100000 / 1200); + System.out.println("merkle leaf node size " + Math.pow(2, 15)); + System.out.println("merkle leaf node size " + (1 << 15)); + } + + @Test + void testMode() { + int mod = (int) Math.pow(2, 3); + long x = HashUtil.hashBytes(UUID.randomUUID().toString().getBytes()); + int xmod = (int) (x % mod + mod); + System.out.println(" mod= " + mod); + System.out.println(x + " " + xmod); + + System.out.println(x + " (int) (x % mod ) =" + (int) (x % mod)); + System.out.println(x + " ( x & (2^n - 1) )= " + (x & (mod - 1))); + + System.out.println(x + " (int) (x % mod + mod) =" + (int) (x % mod + mod)); + System.out.println(x + " ( x & (2^n - 1) + 2^n )= " + ((x & (mod - 1)) + mod)); + + + } + + private static final int[] SCOP_BUCKET_COUNT = new int[15]; + + static { + SCOP_BUCKET_COUNT[0] = 1 << 1; + SCOP_BUCKET_COUNT[1] = 1 << 2; + SCOP_BUCKET_COUNT[2] = 1 << 3; + SCOP_BUCKET_COUNT[3] = 1 << 4; + SCOP_BUCKET_COUNT[4] = 1 << 5; + SCOP_BUCKET_COUNT[5] = 1 << 6; + SCOP_BUCKET_COUNT[6] = 1 << 7; + SCOP_BUCKET_COUNT[7] = 1 << 8; + SCOP_BUCKET_COUNT[8] = 1 << 9; + SCOP_BUCKET_COUNT[9] = 1 << 10; + SCOP_BUCKET_COUNT[10] = 1 << 11; + SCOP_BUCKET_COUNT[11] = 1 << 12; + SCOP_BUCKET_COUNT[12] = 1 << 13; + SCOP_BUCKET_COUNT[13] = 1 << 14; + SCOP_BUCKET_COUNT[14] = 1 << 15; + } + + @Test + public void calacBucketCount() { + int totalCount = 5; + int bucketCount = totalCount / 5; + System.out.println(bucketCount); + int asInt = IntStream.range(0, 15) + .filter(idx -> SCOP_BUCKET_COUNT[idx] > bucketCount) + .peek(System.out::println) + .findFirst() + .orElse(15); + System.out.println(SCOP_BUCKET_COUNT[asInt]); + + } + +} diff --git a/datachecker-extract/.gitignore b/datachecker-extract/.gitignore new file mode 100644 index 0000000..549e00a --- /dev/null +++ b/datachecker-extract/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/datachecker-extract/pom.xml b/datachecker-extract/pom.xml new file mode 100644 index 0000000..3fd952c --- /dev/null +++ b/datachecker-extract/pom.xml @@ -0,0 +1,132 @@ + + + 4.0.0 + + org.opengauss + openGauss-tools-datachecker-performance + 0.0.1 + ../pom.xml + + + datachecker-extract + 0.0.1 + datachecker-extract + jar + datachecker-extract + + + 11 + 0.0.1 + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.opengauss + datachecker-common + ${datachecker.version} + + + org.springframework.boot + spring-boot-starter-log4j2 + + + org.springframework.boot + spring-boot-starter-data-jdbc + + + mysql + mysql-connector-java + provided + + + com.alibaba + druid + + + org.springframework.boot + spring-boot-configuration-processor + true + + + org.projectlombok + lombok + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + net.openhft + zero-allocation-hashing + + + org.springdoc + springdoc-openapi-ui + + + + org.apache.kafka + kafka-streams + + + org.springframework.kafka + spring-kafka + + + + com.google.guava + guava + + + org.springframework.boot + spring-boot-starter-validation + + + com.alibaba + fastjson + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.kafka + spring-kafka-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + mysql + mysql-connector-java + + + + + + + + diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/ExtractApplication.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/ExtractApplication.java new file mode 100644 index 0000000..b3916c5 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/ExtractApplication.java @@ -0,0 +1,17 @@ +package org.opengauss.datachecker.extract; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableAsync; + +@EnableAsync +@EnableFeignClients(basePackages = {"org.opengauss.datachecker.extract.client"}) +@SpringBootApplication +public class ExtractApplication { + + public static void main(String[] args) { + SpringApplication.run(ExtractApplication.class, args); + } + +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/MetaDataCache.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/MetaDataCache.java new file mode 100644 index 0000000..b4d0dfc --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/MetaDataCache.java @@ -0,0 +1,119 @@ +package org.opengauss.datachecker.extract.cache; + +import com.google.common.cache.*; +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.springframework.lang.NonNull; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class MetaDataCache { + private static LoadingCache CACHE = null; + + /** + * Initializing the Metadata Cache Method + */ + public static void initCache() { + CACHE = + CacheBuilder.newBuilder() + //Set the concurrent read/write level based on the number of CPU cores; + .concurrencyLevel(Runtime.getRuntime().availableProcessors()) + // Size of the buffer pool + .maximumSize(Integer.MAX_VALUE) + // Removing a Listener + .removalListener( + (RemovalListener) remove -> log.info("cache: [{}], removed", remove)) + .recordStats() + .build( + // Method of handing a Key that does not exist + new CacheLoader<>() { + @Override + public TableMetadata load(String tableName) { + log.info("cache: [{}], does not exist", tableName); + return null; + } + }); + log.info("initialize table meta data cache"); + } + + /** + * Save to the Cache k v + * + * @param key Metadata key + * @param value Metadata value of table + */ + public static void put(@NonNull String key, TableMetadata value) { + try { + log.info("put in cache:[{}]-[{}]", key, value); + CACHE.put(key, value); + } catch (Exception exception) { + log.error("put in cache exception ", exception); + } + } + + /** + * Batch storage to the cache + * + * @param map map of key,value ,that have some table metadata + */ + public static void putMap(@NonNull Map map) { + try { + CACHE.putAll(map); + map.forEach((key, value) -> log.debug("batch cache deposit:[{},{}]", key, value)); + } catch (Exception exception) { + log.error("batch storage cache exception", exception); + } + } + + /** + * get cache + * + * @param key table name as cached key + */ + public static TableMetadata get(String key) { + try { + return CACHE.get(key); + } catch (Exception exception) { + log.error("get cache exception", exception); + return null; + } + } + + /** + * Obtains all cached key sets + * + * @return keys cached key sets + */ + public static Set getAllKeys() { + try { + return CACHE.asMap().keySet(); + } catch (Exception exception) { + log.error("get cache exception", exception); + return null; + } + } + + /** + * Clear all cache information + */ + public static void removeAll() { + if (Objects.nonNull(CACHE) && CACHE.size() > 0) { + log.info("clear cache information"); + CACHE.cleanUp(); + } + } + + /** + * Specify cache information clearly based on key values + * + * @param key key values of the cache to be cleared + */ + public static void remove(String key) { + if (Objects.nonNull(CACHE) && CACHE.size() > 0) { + log.info("clear cache information"); + CACHE.asMap().remove(key); + } + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/TableExtractStatusCache.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/TableExtractStatusCache.java new file mode 100644 index 0000000..3c2ed3d --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/cache/TableExtractStatusCache.java @@ -0,0 +1,172 @@ +package org.opengauss.datachecker.extract.cache; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.lang.NonNull; +import org.springframework.util.Assert; + +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.IntStream; + +/** + * 

+ * Table data extraction status {@code tableExtractStatusMap}
+ * Cache structure definition {@code Map>} within：
+ * K - Name of the table corresponding to data extraction
+ * T - Sequence number of the current table data extraction and sharding task
+ * V - Completion status of the sharding task for extracting data from the current table.
+ *     When the value is 0,it indicates that the task is not completed，
+ *    other the value is 1,it indicates that the task is completed. the default value is 0.
+ *
+ * + * @author ：wangchao + * @date ：Created in 2022/5/14 + * @since ：11 + */ +@Slf4j +public class TableExtractStatusCache { + /** + * data extraction task completion status + */ + private static final Byte STATUS_COMPLATE = 1; + /** + * Initialization status of a data extraction task + */ + private static final Byte STATUS_INIT = 0; + /** + * Task sequence number. start sequence number + */ + private static final int TASK_ORDINAL_START_INDEX = 1; + /** + * Table data extraction status cache : {@code Map>} + */ + private static final Map> TABLE_EXTRACT_STATUS_MAP = new ConcurrentHashMap<>(); + + + /** + * Table data extraction task status initialization. {code map} is a set of table decomposition tasks. + * + * @param map Initial value of the execution status of the table extraction task. + */ + public static void init(Map map) { + Assert.isTrue(Objects.nonNull(map), Message.INIT_STATUS_PARAM_EMPTY); + map.forEach((table, taskCount) -> { + Map tableStatus = new ConcurrentHashMap<>(); + IntStream.rangeClosed(TASK_ORDINAL_START_INDEX, taskCount) + .forEach(ordinal -> { + tableStatus.put(ordinal, STATUS_INIT); + }); + TABLE_EXTRACT_STATUS_MAP.put(table, tableStatus); + }); + log.info(Message.INIT_STATUS); + } + + + /** + * Updates the execution status of a task in a specified table. + * + * @param tableName table name + * @param ordinal Sequence number of a table data extraction and decomposition task. + */ + public static synchronized void update(@NonNull String tableName, Integer ordinal) { + try { + // the table must exist. + Assert.isTrue(TABLE_EXTRACT_STATUS_MAP.containsKey(tableName), String.format(Message.TABLE_STATUS_NOT_EXIST, tableName)); + + // Obtain the status information corresponding to the current table + // and verify the validity of the task status parameters to be updated. + Map tableStatus = TABLE_EXTRACT_STATUS_MAP.get(tableName); + Assert.isTrue(tableStatus.containsKey(ordinal), String.format(Message.TABLE_ORDINAL_NOT_EXIST, tableName, ordinal)); + + // update status + tableStatus.put(ordinal, STATUS_COMPLATE); + log.info("update tableName : {}, ordinal : {} check completed-status {}", tableName, ordinal, STATUS_COMPLATE); + } catch (Exception exception) { + log.error(Message.UPDATE_STATUS_EXCEPTION, exception); + } + } + + /** + * data extraction status cache message management + */ + interface Message { + String TABLE_STATUS_NOT_EXIST = "The status information of the current table {%s} does not exist. Please initialize it and update it again."; + String TABLE_ORDINAL_NOT_EXIST = "The current table {%s} sequence {%s} task status information does not exist. Please initialize it and update it again."; + String UPDATE_STATUS_EXCEPTION = "Failed to update the task status of the specified table."; + String INIT_STATUS = "Initializing the data extraction task status."; + String INIT_STATUS_PARAM_EMPTY = "The initialization parameter of the data extraction task status cannot be empty."; + } + + + /** + * Check whether the execution status of all tasks corresponding to the current table is complete. + * if true is returned ,all task are completed. + * if false is returned ,at least one task is not completed. + * + * @param tableName table name + */ + public static boolean checkComplated(@NonNull String tableName) { + // check whether the table name exists. + Assert.isTrue(TABLE_EXTRACT_STATUS_MAP.containsKey(tableName), String.format(Message.TABLE_STATUS_NOT_EXIST, tableName)); + return !TABLE_EXTRACT_STATUS_MAP.get(tableName).containsValue(STATUS_INIT); + } + + /** + *

+ * Check whether all tasks whose sequence number is less than {@code ordinal} + * in the current table {@code tableName} are complete. + * if true is returned , all tasks whose sequence number is less than {@code ordinal} + * in the current table {@code tableName} have been completed. + * if false is returned ,at least one task whose sequence number is less than {@code ordinal} + * in the current table {@code tableName} is not completed. + *

+ * + * @param tableName table name + * @param ordinal sequence number of a table splitting task. + * @return + */ + public static boolean checkComplated(@NonNull String tableName, int ordinal) { + // check whether the table name exists. + Assert.isTrue(TABLE_EXTRACT_STATUS_MAP.containsKey(tableName), String.format(Message.TABLE_STATUS_NOT_EXIST, tableName)); + Map tableStatus = TABLE_EXTRACT_STATUS_MAP.get(tableName); + long noComplated = IntStream.range(TASK_ORDINAL_START_INDEX, ordinal) + .filter(idx -> Objects.equals(tableStatus.get(idx), STATUS_INIT)).count(); + log.info("tableName : {}, ordinal : {} check noComplated=[{}]", tableName, ordinal, noComplated); + return noComplated == 0; + } + + /** + * deleting the table extraction status. + * + * @param key table name + */ + public static void remove(@NonNull String key) { + try { + TABLE_EXTRACT_STATUS_MAP.remove(key); + } catch (Exception exception) { + log.error("failed to delete the cache,", exception); + } + } + + /** + * 删除表抽取状态 + */ + public static void removeAll() { + try { + TABLE_EXTRACT_STATUS_MAP.clear(); + } catch (Exception exception) { + log.error("failed to delete the cache,", exception); + } + } + + public static Set getAllKeys() { + try { + return TABLE_EXTRACT_STATUS_MAP.keySet(); + } catch (Exception exception) { + log.error("failed to obtain the cache,", exception); + return null; + } + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/client/CheckingFeignClient.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/client/CheckingFeignClient.java new file mode 100644 index 0000000..63f8ee6 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/client/CheckingFeignClient.java @@ -0,0 +1,56 @@ +package org.opengauss.datachecker.extract.client; + +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.SourceDataLog; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.lang.NonNull; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; + +import javax.validation.constraints.NotEmpty; +import java.util.List; + +/** + * 
+ * create an internal class to declare the API interface of the called party. If the API of the called party is
+ * abnormal ,the exception class is called back for exception declaration.
+ *
+ * The value can be declared in name. The datachecker-check is the service name and directly invokes the system.
+ * Generally,the name uses the Eureka registration information. The Eureka is not introduced.
+ * The URL is configured for invoking.
+ *
+ * + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +@FeignClient(name = "datachecker-check", url = "${spring.check.server-uri}") +public interface CheckingFeignClient { + + /** + * Refresh the execution status of the data extraction table of a specified task. + * + * @param tableName table name + * @param endpoint endpoint enum type {@link org.opengauss.datachecker.common.entry.enums.Endpoint} + */ + @PostMapping("/table/extract/status") + void refushTableExtractStatus(@RequestParam(value = "tableName") @NotEmpty String tableName, + @RequestParam(value = "endpoint") @NonNull Endpoint endpoint); + + /** + * Initializing task status + * + * @param tableNameList table name list + */ + @PostMapping("/table/extract/status/init") + void initTableExtractStatus(@RequestBody @NotEmpty List tableNameList); + + /** + * Incremental verification log notification + * + * @param dataLogList Incremental verification log + */ + @PostMapping("/notify/source/increment/data/logs") + void notifySourceIncrementDataLogs(@RequestBody @NotEmpty List dataLogList); +} \ No newline at end of file diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/AsyncConfig.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/AsyncConfig.java new file mode 100644 index 0000000..d76f78d --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/AsyncConfig.java @@ -0,0 +1,37 @@ +package org.opengauss.datachecker.extract.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * @author wang chao + * @date 2022/5/8 19:17 + * @since 11 + **/ +@Configuration +public class AsyncConfig { + + @Bean("extractThreadExecutor") + public ThreadPoolTaskExecutor doAsyncExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + // Number of core threads, which is the number of threads initialized when the thread pool is created. + executor.setCorePoolSize(Runtime.getRuntime().availableProcessors() * 2); + // Maximum number of threads, maximum number of threads in the thread pool. + executor.setMaxPoolSize(Runtime.getRuntime().availableProcessors() * 4); + // Buffer queue: A queue used to buffer execution tasks. + executor.setQueueCapacity(Integer.MAX_VALUE); + // Allow thread idle time. + executor.setKeepAliveSeconds(60); + // Allow Core Thread Timeout Shutdown + executor.setAllowCoreThreadTimeOut(true); + // Thread pool thread name prefix + executor.setThreadNamePrefix("extract-thread"); + // Deny policy + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + executor.initialize(); + return executor; + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/DruidDataSourceConfig.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/DruidDataSourceConfig.java new file mode 100644 index 0000000..569b902 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/DruidDataSourceConfig.java @@ -0,0 +1,98 @@ +package org.opengauss.datachecker.extract.config; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.support.http.StatViewServlet; +import com.alibaba.druid.support.http.WebStatFilter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.boot.web.servlet.ServletRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.annotation.PostConstruct; +import javax.sql.DataSource; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +@Configuration +public class DruidDataSourceConfig { + + /** + * 
+     *  Add custom Druid data sources to the container,no longer let Spring boot automatically create them.
+     *  Bind the druid data source adttributes in the global configuration file to the com.alibaba.druid.pool.DruidDataSource
+     *  to make them take effect.
+     *  {@code @ConfigurationProperties(prefix="spring.datasource.druid.datasourceone")}: Injects the attribute value
+     *  prefixed with spring.datasource in the global configuration file to the com.alibaba.druid.pool.DruidDataSource
+     *  parameter with the same name.
+     *
+ * + * @return + */ + @Bean("dataSourceOne") + @ConfigurationProperties(prefix = "spring.datasource.druid.datasourceone") + public DataSource druidDataSourceOne() { + return new DruidDataSource(); + } + + + @Bean("jdbcTemplateOne") + public JdbcTemplate jdbcTemplateOne(@Qualifier("dataSourceOne") DataSource dataSourceOne) { + return new JdbcTemplate(dataSourceOne); + } + + /** + * Background monitoring + * Configure the servlet of the Druid monitoring management background. + * There is no web.xml file when the servlet container is built in. Therefore ,the servlet registration mode of + * Spring Boot is used. + * Startup access address : http://localhost:8080/druid/api.html + * + * @return return ServletRegistrationBean + */ + @Bean + public ServletRegistrationBean initServletRegistrationBean() { + + ServletRegistrationBean bean = + new ServletRegistrationBean<>(new StatViewServlet(), "/druid/*"); + // Configuring the account and password + HashMap initParameters = new HashMap<>(); + // Add configuration + // the login key is a fixed loginUsername loginPassword + initParameters.put("loginUsername", "admin"); + initParameters.put("loginPassword", "123456"); + + // if the second parameter is empty,everyone can access it. + initParameters.put("allow", ""); + + // Setting initialization parameters + bean.setInitParameters(initParameters); + return bean; + } + + /** + * Configuring the filter for druid monitoring - web monitoring + * WebStatFilter: used to configure management association monitoring statistice between web and druid data sources. + * + * @return return FilterRegistrationBean + */ + @Bean + public FilterRegistrationBean webStatFilter() { + FilterRegistrationBean bean = new FilterRegistrationBean(); + bean.setFilter(new WebStatFilter()); + + //exclusions: sets the requests to be filtered out so that statistics are not collected. + Map initParams = new HashMap<>(); + // this things don't count. + initParams.put("exclusions", "*.js,*.css,/druid/*,/jdbc/*"); + bean.setInitParameters(initParams); + + //"/*" indicates that all requests are filtered. + bean.setUrlPatterns(Arrays.asList("/*")); + return bean; + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/ExtractConfig.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/ExtractConfig.java new file mode 100644 index 0000000..73daaff --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/ExtractConfig.java @@ -0,0 +1,27 @@ +package org.opengauss.datachecker.extract.config; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.util.JsonObjectUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/23 + * @since ：11 + */ +@Slf4j +@Component +public class ExtractConfig { + + @Autowired + private ExtractProperties extractProperties; + + @PostConstruct + public void initLoad() { + log.info("check config properties [{}]", JsonObjectUtil.format(extractProperties)); + } + +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/ExtractProperties.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/ExtractProperties.java new file mode 100644 index 0000000..9e3fc37 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/ExtractProperties.java @@ -0,0 +1,77 @@ +package org.opengauss.datachecker.extract.config; + +import com.alibaba.fastjson.annotation.JSONType; +import lombok.Data; +import org.opengauss.datachecker.common.entry.enums.DataBaseType; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/29 + * @since ：11 + */ +@Data +@Component +@ConfigurationProperties(prefix = "spring.extract") +@JSONType(orders = {"schema", "databaseType", "endpoint", "debeziumTopic", "debeziumTables"}) +public class ExtractProperties { + + /** + * Database instance : database instance configuration,which cannot be empty + */ + @NotNull(message = "database instance configuration, cannot be empty") + private String schema; + + /** + * Database type: the database type configuration cannot be empty + */ + @NotNull(message = DataBaseType.API_DESCRIPTION + ",database type configuration, cannot be empty") + private DataBaseType databaseType; + + /** + * Endpoint type: Endpoint type,which cannot be empty + */ + @NotNull(message = Endpoint.API_DESCRIPTION + ",endpoint type configuration, cannot be empty") + private Endpoint endpoint; + + /** + * Indicates whether to enable incremental debezium configuration. + * By default,this function is disabled. the default value is false. + */ + @NotNull(message = "whether to enable debezium configuration, which cannot be empty") + private Boolean debeziumEnable; + /** + * Debezium incremental migration verification topic. + * Debezium listens to incremental data in tables and uses a single topic for incremental data management. + * The debezium incremental verification configuration is not checked here. + * If the incremental verification service is required,you need to manually configure it. + */ + private String debeziumTopic; + /** + * Group parameter setting for the debezium incremental migration verification topic + */ + private String debeziumGroupId; + + /** + * Partitions parameter setting for the debezium incremental migration verification topic + */ + private int debeziumTopicPartitions = 1; + /** + * incremental migration table name list + */ + private List debeziumTables; + /** + * debezium incremental migration verification period: 24 x 60 (unit:minute) + */ + private int debeziumTimePeriod; + /** + * debezium incremental migration verification statistics incremental change record count threshold. + * the threshold must be greater than 100. + */ + private int debeziumNumPeriod = 1000; +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/GlobalExtractExceptionHandler.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/GlobalExtractExceptionHandler.java new file mode 100644 index 0000000..b982531 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/GlobalExtractExceptionHandler.java @@ -0,0 +1,59 @@ +package org.opengauss.datachecker.extract.config; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.entry.enums.ResultEnum; +import org.opengauss.datachecker.common.exception.*; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +import javax.servlet.http.HttpServletRequest; + +@Slf4j +@RestControllerAdvice +public class GlobalExtractExceptionHandler extends GlobalCommonExceptionHandler { + private static final String MESSAGE_TEMPLATE = "path:{}, queryParam:[{}] , error:"; + + /** + * service exception handing + */ + @ExceptionHandler(value = ExtractException.class) + public Result extractException(HttpServletRequest request, ExtractException exception) { + logError(request, exception); + return Result.fail(ResultEnum.EXTRACT, exception.getMessage()); + } + + @ExceptionHandler(value = CreateTopicException.class) + public Result createTopicException(HttpServletRequest request, CreateTopicException exception) { + logError(request, exception); + return Result.fail(ResultEnum.CREATE_TOPIC, exception.getMessage()); + } + + @ExceptionHandler(value = ProcessMultipleException.class) + public Result processMultipleException(HttpServletRequest request, ProcessMultipleException exception) { + logError(request, exception); + return Result.fail(ResultEnum.PROCESS_MULTIPLE, exception.getMessage()); + } + + @ExceptionHandler(value = TaskNotFoundException.class) + public Result taskNotFoundException(HttpServletRequest request, TaskNotFoundException exception) { + logError(request, exception); + return Result.fail(ResultEnum.TASK_NOT_FOUND, exception.getMessage()); + } + + @ExceptionHandler(value = TableNotExistException.class) + public Result tableNotExistException(HttpServletRequest request, TableNotExistException exception) { + logError(request, exception); + return Result.fail(ResultEnum.TABLE_NOT_FOUND, exception.getMessage()); + } + + @ExceptionHandler(value = DebeziumConfigException.class) + public Result debeziumConfigException(HttpServletRequest request, DebeziumConfigException exception) { + logError(request, exception); + return Result.fail(ResultEnum.DEBEZIUM_CONFIG_ERROR, exception.getMessage()); + } + + private void logError(HttpServletRequest request, ExtractException exception) { + log.error(MESSAGE_TEMPLATE, request.getRequestURI(), request.getQueryString(), exception); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/KafkaConsumerConfig.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/KafkaConsumerConfig.java new file mode 100644 index 0000000..6bca578 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/KafkaConsumerConfig.java @@ -0,0 +1,99 @@ +package org.opengauss.datachecker.extract.config; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.opengauss.datachecker.common.entry.check.IncrementCheckTopic; +import org.opengauss.datachecker.extract.constants.ExtConstants; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/17 + * @since ：11 + */ +@Slf4j +@Component +@EnableConfigurationProperties(KafkaProperties.class) +public class KafkaConsumerConfig { + + private static final Object LOCK = new Object(); + private static final Map> CONSUMER_MAP = new ConcurrentHashMap<>(); + + @Autowired + private KafkaProperties properties; + + /** + * Obtaining a specified consumer client based on topic. + * + * @param topic topic name + * @param partitions total number of partitions + * @return the topic corresponds to the consumer client. + */ + public KafkaConsumer getKafkaConsumer(String topic, int partitions) { + String consumerKey = topic + "_" + partitions; + KafkaConsumer consumer = CONSUMER_MAP.get(consumerKey); + if (Objects.isNull(consumer)) { + synchronized (LOCK) { + consumer = CONSUMER_MAP.get(consumerKey); + if (Objects.isNull(consumer)) { + consumer = buildKafkaConsumer(); + CONSUMER_MAP.put(consumerKey, consumer); + } + } + } + return consumer; + } + + public KafkaConsumer getDebeziumConsumer(IncrementCheckTopic topic) { + // configuration information + Properties props = new Properties(); + // kafka server address + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, String.join(ExtConstants.DELIMITER, properties.getBootstrapServers())); + // consumer group must be specified + props.put(ConsumerConfig.GROUP_ID_CONFIG, topic.getGroupId()); + // if there are committed offsets in each partition,consumption starts from the submitted offsets. + // when there is no submitted offset,consumption is started from the beginning + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, properties.getConsumer().getAutoOffsetReset()); + // sets the serialization processing class for data keys and values. + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + // creating a kafka consumer instance + return new KafkaConsumer<>(props); + } + + private KafkaConsumer buildKafkaConsumer() { + // configuration information + Properties props = new Properties(); + // kafka server address + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, String.join(ExtConstants.DELIMITER, properties.getBootstrapServers())); + // consumer group must be specified + props.put(ConsumerConfig.GROUP_ID_CONFIG, properties.getConsumer().getGroupId()); + // if there are committed offsets in each partition,consumption starts from the submitted offsets. + // when there is no submitted offset,consumption is started from the beginning + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, properties.getConsumer().getAutoOffsetReset()); + // sets the serialization processing class for data keys and values. + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + // creating a kafka consumer instance + return new KafkaConsumer<>(props); + } + + /** + * clear KafkaConsumer + */ + public void cleanKafkaConsumer() { + CONSUMER_MAP.clear(); + log.info("clear KafkaConsumer"); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/KafkaProducerConfig.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/KafkaProducerConfig.java new file mode 100644 index 0000000..a570dc6 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/KafkaProducerConfig.java @@ -0,0 +1,78 @@ +package org.opengauss.datachecker.extract.config; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/17 + * @since ：11 + */ +@Slf4j +@Component +@EnableConfigurationProperties(KafkaProperties.class) +public class KafkaProducerConfig { + + private static final Object LOCK = new Object(); + private static final Map> PRODUCER_MAP = new ConcurrentHashMap<>(); + + @Autowired + private KafkaProperties properties; + + /** + *Obtaining a specified producer client based on topic. + * + * @param topic topic name + * @return the topic corresponds to the producer client. + */ + public KafkaProducer getKafkaProducer(String topic) { + KafkaProducer producer = PRODUCER_MAP.get(topic); + if (Objects.isNull(producer)) { + synchronized (LOCK) { + producer = PRODUCER_MAP.get(topic); + if (Objects.isNull(producer)) { + producer = buildKafkaProducer(); + PRODUCER_MAP.put(topic, producer); + } + } + } + return producer; + } + + private KafkaProducer buildKafkaProducer() { + // configuration information + Properties props = new Properties(); + // kafka server address + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, properties.getBootstrapServers().stream().collect(Collectors.joining(","))); + props.put(ProducerConfig.ACKS_CONFIG, properties.getProducer().getAcks()); + // sets the serialization processing class for data keys and values. + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, properties.getProducer().getKeySerializer()); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, properties.getProducer().getValueSerializer()); + // creating a kafka producer instance + KafkaProducer producer = new KafkaProducer<>(props); + return producer; + } + + public KafkaProducer getDebeziumKafkaProducer() { + return buildKafkaProducer(); + } + + /** + * clear KafkaProducer + */ + public void cleanKafkaProducer() { + PRODUCER_MAP.clear(); + log.info("clear KafkaProducer"); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/SpringDocConfig.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/SpringDocConfig.java new file mode 100644 index 0000000..e1fd4f9 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/config/SpringDocConfig.java @@ -0,0 +1,73 @@ +package org.opengauss.datachecker.extract.config; + +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.parameters.HeaderParameter; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.springdoc.core.customizers.OpenApiCustomiser; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.util.ReflectionUtils; +import org.springframework.web.servlet.config.annotation.InterceptorRegistration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import java.lang.reflect.Field; +import java.util.List; + +/** + * swagger2 configuration + * http://localhost:8080/swagger-ui/index.html + * + * @author ：wangchao + * @date ：Created in 2022/5/17 + * @since ：11 + */ + +/** + * 2021/8/13 + */ + +@Configuration +public class SpringDocConfig implements WebMvcConfigurer { + @Bean + public OpenAPI mallTinyOpenAPI() { + return new OpenAPI() + .info(new Info() + .title("Data extraction") + .description("Data validation tool data extraction API") + .version("v1.0.0")); + } + + /** + * add global request header parameters. + */ + @Bean + public OpenApiCustomiser customerGlobalHeaderOpenApiCustomiser() { + return openApi -> openApi.getPaths().values().stream().flatMap(pathItem -> pathItem.readOperations().stream()) + .forEach(operation -> { + operation.addParametersItem(new HeaderParameter().$ref("#/components/parameters/myGlobalHeader")); + }); + } + + /** + * general interceptor exclusion settings.all interceptors automatically add springdoc-opapi-related + * resource exclusion information. + * you do not need to add it to the interceptor definition of the application. + */ + @SuppressWarnings("unchecked") + @Override + public void addInterceptors(InterceptorRegistry registry) { + try { + Field registrationsField = FieldUtils.getField(InterceptorRegistry.class, "registrations", true); + List registrations = (List) ReflectionUtils.getField(registrationsField, registry); + if (registrations != null) { + for (InterceptorRegistration interceptorRegistration : registrations) { + interceptorRegistration.excludePathPatterns("/springdoc**/**"); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/constants/ExtConstants.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/constants/ExtConstants.java new file mode 100644 index 0000000..acfed12 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/constants/ExtConstants.java @@ -0,0 +1,13 @@ +package org.opengauss.datachecker.extract.constants; + +import org.opengauss.datachecker.common.constant.Constants; + +public interface ExtConstants { + String PRIMARY_DELIMITER = Constants.PRIMARY_DELIMITER; + String DELIMITER = ","; + + /** + * query result parsing ResultSet data result set,default start index position + */ + int COLUMN_INDEX_FIRST_ZERO = 0; +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractCleanController.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractCleanController.java new file mode 100644 index 0000000..8a54099 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractCleanController.java @@ -0,0 +1,53 @@ +package org.opengauss.datachecker.extract.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.opengauss.datachecker.common.web.Result; +import org.opengauss.datachecker.extract.kafka.KafkaManagerService; +import org.opengauss.datachecker.extract.service.DataExtractService; +import org.opengauss.datachecker.extract.service.MetaDataService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + + +@Tag(name = "Clearing the environment at the extraction endpoint") +@RestController +public class ExtractCleanController { + + @Autowired + private MetaDataService metaDataService; + + @Autowired + private DataExtractService dataExtractService; + + @Autowired + private KafkaManagerService kafkaManagerService; + + /** + * clear the endpoint information and reinitialize the environment. + * + * @return interface invoking result + */ + @Operation(summary = "clear the endpoint information and reinitialize the environment") + @PostMapping("/extract/clean/environment") + Result cleanEnvironment(@RequestParam(name = "processNo") String processNo) { + metaDataService.init(); + dataExtractService.cleanBuildedTask(); + kafkaManagerService.cleanKafka(processNo); + return Result.success(); + } + + @Operation(summary = "clears the task cache information of the current ednpoint") + @PostMapping("/extract/clean/task") + Result cleanTask() { + dataExtractService.cleanBuildedTask(); + return Result.success(); + } + + @Operation(summary = "clear the kafka information of the current endpoint") + @PostMapping("/extract/clean/kafka") + Result cleanKafka() { + kafkaManagerService.cleanKafka(); + return Result.success(); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractController.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractController.java new file mode 100644 index 0000000..f3a64ae --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractController.java @@ -0,0 +1,202 @@ +package org.opengauss.datachecker.extract.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.opengauss.datachecker.common.entry.enums.CheckBlackWhiteMode; +import org.opengauss.datachecker.common.entry.enums.DML; +import org.opengauss.datachecker.common.entry.extract.*; +import org.opengauss.datachecker.common.exception.ProcessMultipleException; +import org.opengauss.datachecker.common.exception.TaskNotFoundException; +import org.opengauss.datachecker.common.web.Result; +import org.opengauss.datachecker.extract.cache.MetaDataCache; +import org.opengauss.datachecker.extract.service.DataExtractService; +import org.opengauss.datachecker.extract.service.MetaDataService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Tag(name = "data extracton service") +@RestController +public class ExtractController { + + @Autowired + private MetaDataService metaDataService; + + @Autowired + private DataExtractService dataExtractService; + + @Operation(summary = "loading database metadata information", + description = "loading database metadata information(including the table name,primary key field information list, and column field information list)") + @GetMapping("/extract/load/database/meta/data") + public Result> queryMetaDataOfSchema() { + Map metaDataMap = metaDataService.queryMetaDataOfSchema(); + MetaDataCache.putMap(metaDataMap); + return Result.success(metaDataMap); + } + + @Operation(summary = "refreshing the block list and trust list") + @PostMapping("/extract/refush/black/white/list") + void refushBlackWhiteList(@RequestParam CheckBlackWhiteMode mode, @RequestBody List tableList) { + metaDataService.refushBlackWhiteList(mode, tableList); + } + + /** + * source endpoint extraction task construction + * + * @param processNo execution process no + * @throws ProcessMultipleException the data extraction service is being executed for the current instance. + * new verification process cannot be enabled. + */ + @Operation(summary = "construction a data extraction task for the current endpoint") + @PostMapping("/extract/build/task/all") + public Result> buildExtractTaskAllTables(@Parameter(name = "processNo", description = "execution process no") + @RequestParam(name = "processNo") String processNo) { + return Result.success(dataExtractService.buildExtractTaskAllTables(processNo)); + } + + /** + * sink endpoint task configuration + * + * @param processNo execution process no + * @param taskList task list + * @throws ProcessMultipleException the data extraction service is being executed for the current instance. + * new verification process cannot be enabled. + */ + @Operation(summary = "sink endpoint task configuration") + @PostMapping("/extract/config/sink/task/all") + Result buildExtractTaskAllTables(@Parameter(name = "processNo", description = "execution process no") + @RequestParam(name = "processNo") String processNo, + @RequestBody List taskList) { + dataExtractService.buildExtractTaskAllTables(processNo, taskList); + return Result.success(); + } + + /** + * full extraction service processing flow: + * 1、create task information based on the table name + * 2、build task thread based on task information + * 2.1 thread pool configuration + * 2.2 task thread construction + * 3、extract data + * 3.1 JDBC extraction, data processing, and data hash calculation + * 3.2 data encapsulation ,pushing kafka + *

+ * execution a table data extraction task + * + * @param processNo execution process no + * @throws TaskNotFoundException if the task data is empty,the TaskNotFoundException exception is thrown. + */ + @Operation(summary = "execute the data extraction task that has been created for the current endpoint") + @PostMapping("/extract/exec/task/all") + public Result execExtractTaskAllTables(@Parameter(name = "processNo", description = "execution process no") + @RequestParam(name = "processNo") String processNo) { + dataExtractService.execExtractTaskAllTables(processNo); + return Result.success(); + } + + /** + * clear the cached task information of the corresponding endpoint and rest the task. + * + * @return interface invoking result + */ + @Operation(summary = " clear the cached task information of the corresponding endpoint and rest the task.") + @PostMapping("/extract/clean/build/task") + public Result cleanBuildedTask() { + dataExtractService.cleanBuildedTask(); + return Result.success(); + } + + /** + * queries information about data extraction tasks in a specified table in the current process. + * + * @param tableName table name + * @return table data extraction task information + */ + @GetMapping("/extract/table/info") + @Operation(summary = "queries information about data extraction tasks in a specified table in the current process.") + Result queryTableInfo(@Parameter(name = "tableName", description = "table name") + @RequestParam(name = "tableName") String tableName) { + return Result.success(dataExtractService.queryTableInfo(tableName)); + } + + /** + * DML statements required to generate a repair report + * + * @param tableName table name + * @param dml dml type + * @param diffSet primary key set + * @return DML statement + */ + @Operation(summary = "DML statements required to generate a repair report") + @PostMapping("/extract/build/repairDML") + Result> buildRepairDml(@NotEmpty(message = "the schema to which the table to be repaired belongs cannot be empty") + @RequestParam(name = "schema") String schema, + @NotEmpty(message = "the name of the table to be repaired belongs cannot be empty") + @RequestParam(name = "tableName") String tableName, + @NotNull(message = "the DML type to be repaired belongs cannot be empty") + @RequestParam(name = "dml") DML dml, + @NotEmpty(message = "the primary key set to be repaired belongs cannot be empty") + @RequestBody Set diffSet) { + return Result.success(dataExtractService.buildRepairDml(schema, tableName, dml, diffSet)); + } + + /** + * querying table data + * + * @param tableName table name + * @param compositeKeySet primary key set + * @return table record data + */ + @Operation(summary = "querying table data") + @PostMapping("/extract/query/table/data") + Result>> queryTableColumnValues(@NotEmpty(message = "the name of the table to be repaired belongs cannot be empty") + @RequestParam(name = "tableName") String tableName, + @NotEmpty(message = "the primary key set to be repaired belongs cannot be empty") + @RequestBody Set compositeKeySet) { + return Result.success(dataExtractService.queryTableColumnValues(tableName, new ArrayList<>(compositeKeySet))); + } + + /** + * creating an incremental extraction task based on data change logs + * + * @param sourceDataLogList data change logs list + * @return interface invoking result + */ + @Operation(summary = "creating an incremental extraction task based on data change logs") + @PostMapping("/extract/increment/logs/data") + Result notifyIncrementDataLogs(@RequestBody @NotNull(message = "数据变更日志不能为空") List sourceDataLogList) { + dataExtractService.buildExtractIncrementTaskByLogs(sourceDataLogList); + dataExtractService.execExtractIncrementTaskByLogs(); + return Result.success(); + } + + @Operation(summary = "query the metadata of the current table structure and perform hash calculation.") + @PostMapping("/extract/query/table/metadata/hash") + Result queryTableMetadataHash(@RequestParam(name = "tableName") String tableName) { + return Result.success(dataExtractService.queryTableMetadataHash(tableName)); + } + + /** + * queries data corresponding to a specified primary key value in a table and performs hash for secondary verification data query. + * + * @param dataLog data change logs + * @return rowdata hash + */ + @Operation(summary = "queries data corresponding to a specified primary key value in a table and performs hash for secondary verification data query.") + @PostMapping("/extract/query/secondary/data/row/hash") + Result> querySecondaryCheckRowData(@RequestBody SourceDataLog dataLog) { + return Result.success(dataExtractService.querySecondaryCheckRowData(dataLog)); + } + + @GetMapping("/extract/query/database/schema") + Result getDatabaseSchema() { + return Result.success(dataExtractService.queryDatabaseSchema()); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractHealthController.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractHealthController.java new file mode 100644 index 0000000..d14b102 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/ExtractHealthController.java @@ -0,0 +1,19 @@ +package org.opengauss.datachecker.extract.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.opengauss.datachecker.common.web.Result; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +@Tag(name = "ExtractHealthController", description = "health check of the data extraction service") +@RestController +public class ExtractHealthController { + + @Operation(summary = "data extraction health check") + @GetMapping("/extract/health") + public Result health() { + return Result.success(); + } + +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/KafkaManagerController.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/KafkaManagerController.java new file mode 100644 index 0000000..3ac6efa --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/controller/KafkaManagerController.java @@ -0,0 +1,123 @@ +package org.opengauss.datachecker.extract.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.opengauss.datachecker.common.entry.extract.RowDataHash; +import org.opengauss.datachecker.common.entry.extract.Topic; +import org.opengauss.datachecker.common.util.IdWorker; +import org.opengauss.datachecker.common.web.Result; +import org.opengauss.datachecker.extract.kafka.KafkaConsumerService; +import org.opengauss.datachecker.extract.kafka.KafkaManagerService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + + +@Tag(name = "KafkaManagerController", description = "Data extraction service: Kafka management service") +@RestController +public class KafkaManagerController { + + @Autowired + private KafkaManagerService kafkaManagerService; + @Autowired + private KafkaConsumerService kafkaConsumerService; + + /** + * Query for specified topic data + * + * @param tableName table Name + * @param partitions topic partition + * @return topic data + */ + @Operation(summary = "Query for specified topic data") + @GetMapping("/extract/query/topic/data") + public Result> queryTopicData(@Parameter(name = "tableName", description = "table Name") + @RequestParam("tableName") String tableName, + @Parameter(name = "partitions", description = "kafka partition number") + @RequestParam("partitions") int partitions) { + return Result.success(kafkaConsumerService.getTopicRecords(tableName, partitions)); + } + + /** + * 查询指定增量topic数据 + * + * @param tableName 表名称 + * @return topic数据 + */ + @Operation(summary = "查询指定增量topic数据") + @GetMapping("/extract/query/increment/topic/data") + public Result> queryIncrementTopicData(@Parameter(name = "tableName", description = "表名称") + @RequestParam("tableName") String tableName) { + return Result.success(kafkaConsumerService.getIncrementTopicRecords(tableName)); + } + + /** + * 根据表名称，创建topic + * + * @param tableName 表名称 + * @param partitions 分区总数 + * @return 创建成功后的topic名称 + */ + @Operation(summary = "根据表名称创建topic", description = "用于测试kafka topic创建") + @PostMapping("/extract/create/topic") + public Result createTopic(@Parameter(name = "tableName", description = "表名称") + @RequestParam("tableName") String tableName, + @Parameter(name = "partitions", description = "kafka分区总数") + @RequestParam("partitions") int partitions) { + String process = IdWorker.nextId36(); + return Result.success(kafkaManagerService.createTopic(process, tableName, partitions)); + } + + /** + * 查询所有的topic名称列表 + * + * @return topic名称列表 + */ + @Operation(summary = "查询当前端点所有的topic名称列表") + @GetMapping("/extract/query/topic") + public Result> queryTopicData() { + return Result.success(kafkaManagerService.getAllTopic()); + } + + @Operation(summary = "查询指定表名的Topic信息") + @GetMapping("/extract/topic/info") + public Result queryTopicInfo(@Parameter(name = "tableName", description = "表名称") + @RequestParam(name = "tableName") String tableName) { + return Result.success(kafkaManagerService.getTopic(tableName)); + } + + @Operation(summary = "查询指定表名的Topic信息") + @GetMapping("/extract/increment/topic/info") + public Result getIncrementTopicInfo(@Parameter(name = "tableName", description = "表名称") + @RequestParam(name = "tableName") String tableName) { + return Result.success(kafkaManagerService.getIncrementTopicInfo(tableName)); + } + + @Operation(summary = "清理所有数据抽取相关topic", description = "清理kafka中 前缀TOPIC_EXTRACT_Endpoint_process_ 的所有Topic") + @PostMapping("/extract/delete/topic/history") + public Result deleteTopic(@Parameter(name = "processNo", description = "校验流程号") + @RequestParam(name = "processNo") String processNo) { + kafkaManagerService.deleteTopic(processNo); + return Result.success(); + } + + @Operation(summary = "清理所有数据抽取相关topic", description = "清理kafka中 前缀TOPIC_EXTRACT_Endpoint_ 的所有Topic") + @PostMapping("/extract/super/delete/topic/history") + public Result deleteTopic() { + kafkaManagerService.deleteTopic(); + return Result.success(); + } + + @Operation(summary = "删除kafka中指定topic", description = "删除kafka中指定topic") + @PostMapping("/extract/delete/topic") + public Result deleteTopicHistory(@Parameter(name = "topicName", description = "topic名称") + @RequestParam(name = "topicName") String topicName) { + kafkaManagerService.deleteTopicByName(topicName); + return Result.success(); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/DataBaseMetaDataDAOImpl.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/DataBaseMetaDataDAOImpl.java new file mode 100644 index 0000000..060b0af --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/DataBaseMetaDataDAOImpl.java @@ -0,0 +1,172 @@ +package org.opengauss.datachecker.extract.dao; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.constant.Constants; +import org.opengauss.datachecker.common.entry.enums.CheckBlackWhiteMode; +import org.opengauss.datachecker.common.entry.enums.DataBaseMeta; +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.opengauss.datachecker.common.entry.enums.ColumnKey; +import org.opengauss.datachecker.common.util.EnumUtil; +import org.opengauss.datachecker.extract.config.ExtractProperties; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCountCallbackHandler; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.opengauss.datachecker.extract.constants.ExtConstants.COLUMN_INDEX_FIRST_ZERO; + +@Slf4j +@Component +@RequiredArgsConstructor +public class DataBaseMetaDataDAOImpl implements MetaDataDAO { + + private static final AtomicReference MODE_REF = new AtomicReference<>(CheckBlackWhiteMode.NONE); + private static final AtomicReference> WHITE_REF = new AtomicReference<>(); + private static final AtomicReference> BLACK_REF = new AtomicReference<>(); + + protected final JdbcTemplate JdbcTemplateOne; + + private final ExtractProperties extractProperties; + + @Override + public void resetBlackWhite(CheckBlackWhiteMode mode, List tableList) { + MODE_REF.set(mode); + if (Objects.equals(mode, CheckBlackWhiteMode.WHITE)) { + WHITE_REF.set(tableList); + } else if (Objects.equals(mode, CheckBlackWhiteMode.BLACK)) { + BLACK_REF.set(tableList); + } else { + WHITE_REF.getAcquire().clear(); + BLACK_REF.getAcquire().clear(); + } + } + + @Override + public List queryTableMetadata() { + final List tableNameList = new ArrayList<>(); + String sql = MetaSqlMapper.getMetaSql(extractProperties.getDatabaseType(), DataBaseMeta.TABLE); + JdbcTemplateOne.query(sql, ps -> ps.setString(1, getSchema()), new RowCountCallbackHandler() { + @Override + protected void processRow(ResultSet rs, int rowNum) throws SQLException { + tableNameList.add(rs.getString(1)); + } + }); + return getAllTableCount(filterTableListByBlackWhite(tableNameList)); + } + + private List filterTableListByBlackWhite(List tableNameList) { + if (Objects.equals(MODE_REF.get(), CheckBlackWhiteMode.WHITE)) { + final List whiteList = WHITE_REF.get(); + if (CollectionUtils.isEmpty(whiteList)) { + return tableNameList; + } + return tableNameList.stream().filter(whiteList::contains).collect(Collectors.toList()); + } else if (Objects.equals(MODE_REF.get(), CheckBlackWhiteMode.BLACK)) { + final List blackList = BLACK_REF.get(); + if (CollectionUtils.isEmpty(blackList)) { + return tableNameList; + } + return tableNameList.stream().filter(table -> !blackList.contains(table)).collect(Collectors.toList()); + } else { + return tableNameList; + } + } + + private List filterBlackWhiteList(List tableMetaList) { + if (Objects.equals(MODE_REF.get(), CheckBlackWhiteMode.WHITE)) { + final List whiteList = WHITE_REF.get(); + if (CollectionUtils.isEmpty(whiteList)) { + return tableMetaList; + } + return tableMetaList.stream().filter(table -> whiteList.contains(table.getTableName())).collect(Collectors.toList()); + } else if (Objects.equals(MODE_REF.get(), CheckBlackWhiteMode.BLACK)) { + final List blackList = BLACK_REF.get(); + if (CollectionUtils.isEmpty(blackList)) { + return tableMetaList; + } + return tableMetaList.stream().filter(table -> !blackList.contains(table.getTableName())).collect(Collectors.toList()); + } else { + return tableMetaList; + } + } + + @Override + public List queryTableMetadataFast() { + List tableMetadata = new ArrayList<>(); + String sql = MetaSqlMapper.getMetaSql(extractProperties.getDatabaseType(), DataBaseMeta.TABLE); + JdbcTemplateOne.query(sql, ps -> ps.setString(1, getSchema()), new RowCountCallbackHandler() { + @Override + protected void processRow(ResultSet rs, int rowNum) throws SQLException { + final TableMetadata metadata = new TableMetadata() + .setTableName(rs.getString(1)) + .setTableRows(rs.getLong(2)); + log.debug("queryTableMetadataFast {}", metadata.toString()); + tableMetadata.add(metadata); + } + }); + return filterBlackWhiteList(tableMetadata); + } + + private List getAllTableCount(List tableNameList) { + final List tableMetadata = new ArrayList<>(); + String sqlQueryTableRowCount = MetaSqlMapper.getTableCount(); + final String schema = getSchema(); + tableNameList.stream().forEach(tableName -> { + final Long rowCount = JdbcTemplateOne.queryForObject(String.format(sqlQueryTableRowCount, schema, tableName), Long.class); + tableMetadata.add(new TableMetadata().setTableName(tableName).setTableRows(rowCount)); + }); + return tableMetadata; + } + + @Override + public List queryColumnMetadata(String tableName) { + return queryColumnMetadata(Arrays.asList(tableName)); + } + + @Override + public List queryColumnMetadata(List tableNames) { + Map map = new HashMap<>(Constants.InitialCapacity.MAP); + map.put("tableNames", tableNames); + map.put("databaseSchema", getSchema()); + NamedParameterJdbcTemplate jdbc = new NamedParameterJdbcTemplate(JdbcTemplateOne); + String sql = MetaSqlMapper.getMetaSql(extractProperties.getDatabaseType(), DataBaseMeta.COLUMN); + return jdbc.query(sql, map, new RowMapper() { + int columnIndex = COLUMN_INDEX_FIRST_ZERO; + + @Override + public ColumnsMetaData mapRow(ResultSet rs, int rowNum) throws SQLException { + + ColumnsMetaData columnsMetaData = new ColumnsMetaData() + .setTableName(rs.getString(++columnIndex)) + .setColumnName(rs.getString(++columnIndex)) + .setOrdinalPosition(rs.getInt(++columnIndex)) + .setDataType(rs.getString(++columnIndex)) + .setColumnType(rs.getString(++columnIndex)) + .setColumnKey(EnumUtil.valueOf(ColumnKey.class, rs.getString(++columnIndex))); + columnIndex = COLUMN_INDEX_FIRST_ZERO; + return columnsMetaData; + } + }); + } + + /** + * 动态获取当前数据源的schema信息 + * + * @return + */ + private String getSchema() { + return extractProperties.getSchema(); + } + + +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/MetaDataDAO.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/MetaDataDAO.java new file mode 100644 index 0000000..43489aa --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/MetaDataDAO.java @@ -0,0 +1,48 @@ +package org.opengauss.datachecker.extract.dao; + +import org.opengauss.datachecker.common.entry.enums.CheckBlackWhiteMode; +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; + +import java.util.List; + +public interface MetaDataDAO { + /** + * 重置黑白名单 + * + * @param mode 黑白名单模式{@link CheckBlackWhiteMode} + * @param tableList 表名称列表 + */ + void resetBlackWhite(CheckBlackWhiteMode mode, List tableList); + + /** + * 查询表元数据 + * + * @return 返回表元数据信息 + */ + List queryTableMetadata(); + + /** + * 快速查询表元数据 -直接从information_schema获取 + * + * @return 返回表元数据信息 + */ + List queryTableMetadataFast(); + + /** + * 查询表对应列元数据信息 + * + * @param tableName 表名称 + * @return 列元数据信息 + */ + List queryColumnMetadata(String tableName); + + /** + * 查询表对应列元数据信息 + * + * @param tableNames 表名称 + * @return 列元数据信息 + */ + List queryColumnMetadata(List tableNames); + +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/MetaSqlMapper.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/MetaSqlMapper.java new file mode 100644 index 0000000..083582b --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dao/MetaSqlMapper.java @@ -0,0 +1,72 @@ +package org.opengauss.datachecker.extract.dao; + +import org.opengauss.datachecker.common.entry.enums.DataBaseMeta; +import org.opengauss.datachecker.common.entry.enums.DataBaseType; +import org.springframework.util.Assert; + +import java.util.HashMap; +import java.util.Map; + +public class MetaSqlMapper { + + public static String getTableCount() { + return "select count(1) rowCount from %s.%s"; + } + + interface DataBaseMySql { + String TABLE_METADATA_SQL = "select table_name tableName , table_rows tableRows from information_schema.tables WHERE table_schema=?"; + String TABLES_COLUMN_META_DATA_SQL = "select table_name tableName ,column_name columnName, ordinal_position ordinalPosition, data_type dataType, column_type columnType,column_key columnKey from information_schema.columns where table_schema=:databaseSchema and table_name in (:tableNames)"; + } + + interface DataBaseOpenGauss { + String TABLE_METADATA_SQL = "select table_name tableName , 0 tableRows from information_schema.tables WHERE table_schema=? and TABLE_TYPE='BASE TABLE';"; + String TABLES_COLUMN_META_DATA_SQL = "select c.table_name tableName ,c.column_name columnName, c.ordinal_position ordinalPosition, c.data_type dataType , c.data_type columnType,pkc.column_key\n" + + " from information_schema.columns c \n" + + " left join (\n" + + " select kcu.table_name,kcu.column_name,'PRI' column_key\n" + + " from information_schema.key_column_usage kcu \n" + + " WHERE kcu.constraint_name in (\n" + + " select constraint_name from information_schema.table_constraints tc where tc.constraint_schema=:databaseSchema and tc.constraint_type='PRIMARY KEY'\n" + + " )\n" + + " ) pkc on c.table_name=pkc.table_name and c.column_name=pkc.column_name\n" + + " where c.table_schema =:databaseSchema and c.table_name in (:tableNames)"; + } + + interface DataBaseO { + String TABLE_METADATA_SQL = ""; + String TABLES_COLUMN_META_DATA_SQL = ""; + } + + private static final Map> DATABASE_META_MAPPER = new HashMap<>(); + + + static { + Map dataBaseMySql = new HashMap<>(); + dataBaseMySql.put(DataBaseMeta.TABLE, DataBaseMySql.TABLE_METADATA_SQL); + dataBaseMySql.put(DataBaseMeta.COLUMN, DataBaseMySql.TABLES_COLUMN_META_DATA_SQL); + DATABASE_META_MAPPER.put(DataBaseType.MS, dataBaseMySql); + + Map dataBaseOpenGauss = new HashMap<>(); + dataBaseOpenGauss.put(DataBaseMeta.TABLE, DataBaseOpenGauss.TABLE_METADATA_SQL); + dataBaseOpenGauss.put(DataBaseMeta.COLUMN, DataBaseOpenGauss.TABLES_COLUMN_META_DATA_SQL); + DATABASE_META_MAPPER.put(DataBaseType.OG, dataBaseOpenGauss); + + Map databaseO = new HashMap<>(); + databaseO.put(DataBaseMeta.TABLE, DataBaseO.TABLE_METADATA_SQL); + databaseO.put(DataBaseMeta.COLUMN, DataBaseO.TABLES_COLUMN_META_DATA_SQL); + DATABASE_META_MAPPER.put(DataBaseType.O, databaseO); + + } + + /** + * 根据数据库类型 以及当前要执行的元数据查询类型 返回对应的元数据执行语句 + * + * @param dataBaseType 数据库类型 + * @param dataBaseMeta 数据库元数据 + * @return + */ + public static String getMetaSql(DataBaseType dataBaseType, DataBaseMeta dataBaseMeta) { + Assert.isTrue(DATABASE_META_MAPPER.containsKey(dataBaseType), "数据库类型不匹配"); + return DATABASE_META_MAPPER.get(dataBaseType).get(dataBaseMeta); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/BatchDeleteDmlBuilder.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/BatchDeleteDmlBuilder.java new file mode 100644 index 0000000..5404b0d --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/BatchDeleteDmlBuilder.java @@ -0,0 +1,71 @@ +package org.opengauss.datachecker.extract.dml; + +import org.apache.commons.lang3.StringUtils; +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.springframework.lang.NonNull; +import org.springframework.util.Assert; + +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/13 + * @since ：11 + */ +public class BatchDeleteDmlBuilder extends DmlBuilder { + + /** + * 构建 Schema + * + * @param schema Schema + * @return DeleteDMLBuilder 构建器 + */ + public BatchDeleteDmlBuilder schema(@NotNull String schema) { + super.buildSchema(schema); + return this; + } + /** + * 构建 tableName + * + * @param tableName tableName + * @return DeleteDMLBuilder 构建器 + */ + public BatchDeleteDmlBuilder tableName(@NotNull String tableName) { + super.buildTableName(tableName); + return this; + } + + /** + * 生成单一主键字段 delete from schema.table where pk in (参数...) 条件语句 + * + * @param primaryMeta 主键元数据 + * @return DeleteDMLBuilder 构建器 + */ + public BatchDeleteDmlBuilder conditionPrimary(@NonNull ColumnsMetaData primaryMeta) { + Assert.isTrue(StringUtils.isNotEmpty(primaryMeta.getColumnName()), "表元数据主键字段名称为空"); + this.condition = primaryMeta.getColumnName().concat(IN); + return this; + } + /** + * 构建复合主键参数的条件查询语句

+ * select columns... from table where (pk1,pk2) in ((pk1_val,pk2_val),(pk1_val,pk2_val))

+ * + * @param primaryMeta + * @return SelectDMLBuilder构建器 + */ + public BatchDeleteDmlBuilder conditionCompositePrimary(@NonNull List primaryMeta) { + this.condition = buildConditionCompositePrimary(primaryMeta).concat(IN); + return this; + } + + public String build() { + StringBuffer sb = new StringBuffer(); + sb.append(Fragment.DELETE).append(Fragment.FROM) + .append(schema).append(Fragment.LINKER).append(tableName) + .append(Fragment.WHERE).append(condition) + .append(Fragment.END) + ; + return sb.toString(); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/DeleteDmlBuilder.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/DeleteDmlBuilder.java new file mode 100644 index 0000000..d7514f8 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/DeleteDmlBuilder.java @@ -0,0 +1,118 @@ +package org.opengauss.datachecker.extract.dml; + +import org.apache.commons.lang3.StringUtils; +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.extract.constants.ExtConstants; +import org.springframework.lang.NonNull; +import org.springframework.util.Assert; + +import javax.validation.constraints.NotNull; +import java.util.List; +import java.util.stream.IntStream; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/13 + * @since ：11 + */ +public class DeleteDmlBuilder extends DmlBuilder { + + /** + * 构建 Schema + * + * @param schema Schema + * @return DeleteDMLBuilder 构建器 + */ + public DeleteDmlBuilder schema(@NotNull String schema) { + super.buildSchema(schema); + return this; + } + + /** + * 构建 tableName + * + * @param tableName tableName + * @return DeleteDMLBuilder 构建器 + */ + public DeleteDmlBuilder tableName(@NotNull String tableName) { + super.buildTableName(tableName); + return this; + } + + /** + * 生成单一主键字段 delete from schema.table where pk = 参数 条件语句 + * + * @param primaryMeta 主键元数据 + * @return DeleteDMLBuilder 构建器 + */ + public DeleteDmlBuilder condition(@NonNull ColumnsMetaData primaryMeta, String value) { + Assert.isTrue(StringUtils.isNotEmpty(primaryMeta.getColumnName()), "表元数据主键字段名称为空"); + if (DIGITAL.contains(primaryMeta.getDataType())) { + this.condition = primaryMeta.getColumnName().concat(EQUAL).concat(value); + } else { + this.condition = primaryMeta.getColumnName().concat(EQUAL) + .concat(SINGLE_QUOTES).concat(value).concat(SINGLE_QUOTES); + } + return this; + } + + /** + * 构建复合主键参数的条件 delete语句

+ * delete from schema.table where pk1 = pk1_val and pk2 = pk2_val

+ * + * @param compositeKey 复合主键 + * @param primaryMetas 主键元数据 + * @return SelectDMLBuilder 构建器 + */ + public DeleteDmlBuilder conditionCompositePrimary(String compositeKey, List primaryMetas) { + this.condition = buildCondition(compositeKey, primaryMetas); + return this; + } + + /** + * 构建主键过滤（where）条件

+ * pk = pk_value

+ * or

+ * pk = 'pk_value'

+ * + * @param compositeKey 复合主键 + * @param primaryMetas 主键元数据 + * @return 返回主键where条件 + */ + public String buildCondition(String compositeKey, List primaryMetas) { + final String[] split = compositeKey.split(ExtConstants.PRIMARY_DELIMITER); + StringBuffer conditionBuffer = new StringBuffer(); + if (split.length == primaryMetas.size()) { + IntStream.range(0, primaryMetas.size()).forEach(idx -> { + String condition = ""; + final ColumnsMetaData mate = primaryMetas.get(idx); + if (idx > 0) { + condition = condition.concat(AND); + } + if (DIGITAL.contains(mate.getDataType())) { + condition = condition.concat(mate.getColumnName()) + .concat(EQUAL) + .concat(split[idx]); + } else { + condition = condition.concat(mate.getColumnName()) + .concat(EQUAL) + .concat(SINGLE_QUOTES) + .concat(split[idx]) + .concat(SINGLE_QUOTES); + } + conditionBuffer.append(condition); + }); + } + return conditionBuffer.toString(); + } + + public String build() { + StringBuffer sb = new StringBuffer(); + sb.append(Fragment.DELETE).append(Fragment.FROM) + .append(schema).append(Fragment.LINKER).append(tableName) + .append(Fragment.WHERE).append(condition) + .append(Fragment.END) + ; + return sb.toString(); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/DmlBuilder.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/DmlBuilder.java new file mode 100644 index 0000000..c27ee2f --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/DmlBuilder.java @@ -0,0 +1,100 @@ +package org.opengauss.datachecker.extract.dml; + +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; + +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/13 + * @since ：11 + */ +public class DmlBuilder { + + protected static final String DELIMITER = ","; + protected static final String LEFT_BRACKET = "("; + protected static final String RIGHT_BRACKET = ")"; + protected static final String IN = " in ( :primaryKeys )"; + protected static final String SINGLE_QUOTES = "'"; + protected static final String EQUAL = " = "; + protected static final String AND = " and "; + public static final String PRIMARY_KEYS = "primaryKeys"; + /** + * mysql dataType + */ + protected final List DIGITAL = List.of("int", "tinyint", "smallint", "mediumint", "bit", "bigint", "double", "float", "decimal"); + + + protected String columns; + protected String columnsValue; + protected String schema; + protected String tableName; + protected String condition; + protected String conditionValue; + + + /** + * 构建SQL column 语句片段 + * + * @param columnsMetas 字段元数据 + * @return SQL column 语句片段 + */ + protected void buildColumns(@NotNull List columnsMetas) { + this.columns = columnsMetas.stream() + .map(ColumnsMetaData::getColumnName) + .collect(Collectors.joining(DELIMITER)); + } + + protected void buildSchema(@NotNull String schema) { + this.schema = schema; + } + + protected void buildTableName(@NotNull String tableName) { + this.tableName = tableName; + } + + protected String buildConditionCompositePrimary(List primaryMetas) { + return primaryMetas.stream() + .map(ColumnsMetaData::getColumnName) + .collect(Collectors.joining(DELIMITER, LEFT_BRACKET, RIGHT_BRACKET)); + } + + public List columnsValueList(@NotNull Map columnsValue, @NotNull List columnsMetaList) { + List valueList = new ArrayList<>(); + columnsMetaList.forEach(columnMeta -> { + if (DIGITAL.contains(columnMeta.getDataType())) { + valueList.add(columnsValue.get(columnMeta.getColumnName())); + } else { + String value = columnsValue.get(columnMeta.getColumnName()); + if (Objects.isNull(value)) { + valueList.add("null"); + } else { + valueList.add(SINGLE_QUOTES.concat(value).concat(SINGLE_QUOTES)); + } + } + }); + return valueList; + } + + interface Fragment { + String DML_INSERT = "insert into #schema.#tablename (#columns) value (#value);"; + String DML_REPLACE = "replace into #schema.#tablename (#columns) value (#value);"; + String SELECT = "select "; + String DELETE = "delete "; + String FROM = " from "; + String WHERE = " where "; + String SPACE = " "; + String END = ";"; + String LINKER = "."; + + String SCHEMA = "#schema"; + String TABLE_NAME = "#tablename"; + String COLUMNS = "#columns"; + String VALUE = "#value"; + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/InsertDmlBuilder.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/InsertDmlBuilder.java new file mode 100644 index 0000000..fcce89f --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/InsertDmlBuilder.java @@ -0,0 +1,74 @@ +package org.opengauss.datachecker.extract.dml; + +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; + +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/14 + * @since ：11 + */ +public class InsertDmlBuilder extends DmlBuilder { + + + /** + * 构建 Schema + * + * @param schema Schema + * @return InsertDMLBuilder 构建器 + */ + public InsertDmlBuilder schema(@NotNull String schema) { + super.buildSchema(schema); + return this; + } + + /** + * 构建 tableName + * + * @param tableName tableName + * @return InsertDMLBuilder 构建器 + */ + public InsertDmlBuilder tableName(@NotNull String tableName) { + super.buildTableName(tableName); + return this; + } + + /** + * 构建SQL column 语句片段 + * + * @param columnsMetas 字段元数据 + * @return InsertDMLBuilder 构建器 + */ + public InsertDmlBuilder columns(@NotNull List columnsMetas) { + this.columns = columnsMetas.stream() + .map(ColumnsMetaData::getColumnName) + .collect(Collectors.joining(DELIMITER)); + return this; + } + + /** + * 构建SQL column value 语句片段 + * + * @param columnsMetaList 字段元数据 + * @return InsertDMLBuilder 构建器 + */ + public InsertDmlBuilder columnsValue(@NotNull Map columnsValue, @NotNull List columnsMetaList) { + List valueList = new ArrayList<>(columnsValueList(columnsValue, columnsMetaList)); + this.columnsValue = String.join(DELIMITER, valueList); + return this; + } + + public String build() { + return Fragment.DML_INSERT.replace(Fragment.SCHEMA, schema) + .replace(Fragment.TABLE_NAME, tableName) + .replace(Fragment.COLUMNS, columns) + .replace(Fragment.VALUE, columnsValue) + ; + } + +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/ReplaceDmlBuilder.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/ReplaceDmlBuilder.java new file mode 100644 index 0000000..1b7a2da --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/ReplaceDmlBuilder.java @@ -0,0 +1,75 @@ +package org.opengauss.datachecker.extract.dml; + +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; + +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/14 + * @since ：11 + */ +public class ReplaceDmlBuilder extends DmlBuilder { + + + /** + * 构建 Schema + * + * @param schema Schema + * @return InsertDMLBuilder 构建器 + */ + public ReplaceDmlBuilder schema(@NotNull String schema) { + super.buildSchema(schema); + return this; + } + + /** + * 构建 tableName + * + * @param tableName tableName + * @return InsertDMLBuilder 构建器 + */ + public ReplaceDmlBuilder tableName(@NotNull String tableName) { + super.buildTableName(tableName); + return this; + } + + /** + * 构建SQL column 语句片段 + * + * @param columnsMetas 字段元数据 + * @return InsertDMLBuilder 构建器 + */ + public ReplaceDmlBuilder columns(@NotNull List columnsMetas) { + this.columns = columnsMetas.stream() + .map(ColumnsMetaData::getColumnName) + .collect(Collectors.joining(DELIMITER)); + return this; + } + + /** + * 构建SQL column value 语句片段 + * + * @param columnsMetaList 字段元数据 + * @return InsertDMLBuilder 构建器 + */ + public ReplaceDmlBuilder columnsValue(@NotNull Map columnsValue, @NotNull List columnsMetaList) { + List valueList = new ArrayList<>(columnsValueList(columnsValue, columnsMetaList)); + this.columnsValue = String.join(DELIMITER, valueList); + return this; + } + + + public String build() { + return Fragment.DML_REPLACE.replace(Fragment.SCHEMA, schema) + .replace(Fragment.TABLE_NAME, tableName) + .replace(Fragment.COLUMNS, columns) + .replace(Fragment.VALUE, columnsValue) + ; + } + +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/SelectDmlBuilder.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/SelectDmlBuilder.java new file mode 100644 index 0000000..8e14cc2 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/dml/SelectDmlBuilder.java @@ -0,0 +1,116 @@ +package org.opengauss.datachecker.extract.dml; + + +import org.apache.commons.lang3.StringUtils; +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.extract.constants.ExtConstants; +import org.springframework.lang.NonNull; +import org.springframework.util.Assert; + +import javax.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/13 + * @since ：11 + */ +public class SelectDmlBuilder extends DmlBuilder { + + /** + * 构建SQL column 语句片段 + * + * @param columnsMetas 字段元数据 + * @return SelectDMLBuilder构建器 + */ + public SelectDmlBuilder columns(@NotNull List columnsMetas) { + super.buildColumns(columnsMetas); + return this; + } + + /** + * 构建 Schema + * + * @param schema Schema + * @return SelectDMLBuilder构建器 + */ + public SelectDmlBuilder schema(@NotNull String schema) { + super.buildSchema(schema); + return this; + } + + /** + * 生成单一主键字段 select columns... from where pk in (参数...) 条件语句 + * + * @param primaryMeta 主键元数据 + * @return SelectDMLBuilder构建器 + */ + public SelectDmlBuilder conditionPrimary(@NonNull ColumnsMetaData primaryMeta) { + Assert.isTrue(StringUtils.isNotEmpty(primaryMeta.getColumnName()), "表元数据主键字段名称为空"); + this.condition = primaryMeta.getColumnName().concat(IN); + return this; + } + + /** + * 构建复合主键参数的条件查询语句

+ * select columns... from table where (pk1,pk2) in ((pk1_val,pk2_val),(pk1_val,pk2_val))

+ * + * @param primaryMeta + * @return SelectDMLBuilder构建器 + */ + public SelectDmlBuilder conditionCompositePrimary(@NonNull List primaryMeta) { + this.condition = buildConditionCompositePrimary(primaryMeta).concat(IN); + return this; + } + + + + /** + * 构建复合主键参数的条件查询语句 value 参数

+ * select columns... from table where (pk1,pk2) in ((pk1_val,pk2_val),(pk1_val,pk2_val))

+ * + * @param primaryMetas 主键元数据信息 + * @param compositeKeys 主键值列表 + * @return SelectDMLBuilder构建器 + */ + public List conditionCompositePrimaryValue(@NonNull List primaryMetas, List compositeKeys) { + List batchParam = new ArrayList<>(); + final int size = primaryMetas.size(); + compositeKeys.forEach(compositeKey -> { + final String[] split = compositeKey.split(ExtConstants.PRIMARY_DELIMITER); + if (split.length == size) { + Object[] values = new Object[size]; + IntStream.range(0, primaryMetas.size()).forEach(idx -> { + values[idx] = split[idx]; + }); + batchParam.add(values); + } + }); + return batchParam; + } + + /** + * 构建 tableName + * + * @param tableName tableName + * @return SelectDMLBuilder构建器 + */ + public SelectDmlBuilder tableName(@NotNull String tableName) { + super.buildTableName(tableName); + return this; + } + + + public String build() { + StringBuffer sb = new StringBuffer(); + sb.append(Fragment.SELECT).append(columns).append(Fragment.FROM) + .append(schema).append(Fragment.LINKER).append(tableName) + .append(Fragment.WHERE).append(condition) + .append(Fragment.END) + ; + return sb.toString(); + } + +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractService.java new file mode 100644 index 0000000..c9f019c --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractService.java @@ -0,0 +1,116 @@ +package org.opengauss.datachecker.extract.service; + +import org.opengauss.datachecker.common.entry.enums.DML; +import org.opengauss.datachecker.common.entry.extract.ExtractTask; +import org.opengauss.datachecker.common.entry.extract.RowDataHash; +import org.opengauss.datachecker.common.entry.extract.SourceDataLog; +import org.opengauss.datachecker.common.entry.extract.TableMetadataHash; +import org.opengauss.datachecker.common.exception.ProcessMultipleException; +import org.opengauss.datachecker.common.exception.TaskNotFoundException; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @author wang chao + * @description 数据抽取服务 + * @date 2022/5/8 19:27 + * @since 11 + **/ +public interface DataExtractService { + + /** + * 抽取任务构建 + * + * @param processNo 执行进程编号 + * @return 指定processNo下 构建抽取任务集合 + * @throws ProcessMultipleException 当前实例正在执行数据抽取服务，不能重新开启新的校验。 + */ + List buildExtractTaskAllTables(String processNo) throws ProcessMultipleException; + + /** + * 宿端任务配置 + * + * @param processNo 执行进程编号 + * @param taskList 任务列表 + * @throws ProcessMultipleException 前实例正在执行数据抽取服务，不能重新开启新的校验。 + */ + void buildExtractTaskAllTables(String processNo, List taskList) throws ProcessMultipleException; + + /** + * 执行表数据抽取任务 + * + * @param processNo 执行进程编号 + * @throws TaskNotFoundException 任务数据为空，则抛出异常 TaskNotFoundException + */ + void execExtractTaskAllTables(String processNo) throws TaskNotFoundException; + + /** + * 清理当前构建任务 + */ + void cleanBuildedTask(); + + /** + * 查询当前流程下，指定名称的详细任务信息 + * + * @param taskName 任务名称 + * @return 任务详细信息，若不存在返回{@code null} + */ + ExtractTask queryTableInfo(String taskName); + + /** + * 生成修复报告的DML语句 + * + * @param schema schema信息 + * @param tableName 表名 + * @param dml dml 类型 + * @param diffSet 待生成主键集合 + * @return DML语句 + */ + List buildRepairDml(String schema, String tableName, DML dml, Set diffSet); + + /** + * 查询表数据 + * + * @param tableName 表名称 + * @param compositeKeySet 复核主键集合 + * @return 主键对应表数据 + */ + List> queryTableColumnValues(String tableName, List compositeKeySet); + + /** + * 根据数据变更日志 构建增量抽取任务 + * + * @param sourceDataLogs 数据变更日志 + */ + void buildExtractIncrementTaskByLogs(List sourceDataLogs); + + /** + * 执行增量校验数据抽取 + */ + void execExtractIncrementTaskByLogs(); + + /** + * 查询当前表结构元数据信息，并进行Hash + * + * @param tableName 表名称 + * @return 表结构Hash + */ + TableMetadataHash queryTableMetadataHash(String tableName); + + /** + * 查询表指定PK列表数据，并进行Hash 用于二次校验数据查询 + * + * @param dataLog 数据日志 + * @return rowdata hash + */ + List querySecondaryCheckRowData(SourceDataLog dataLog); + + /** + * 查询当前链接数据库 的schema + * + * @return 数据库的schema + */ + String queryDatabaseSchema(); +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractServiceImpl.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractServiceImpl.java new file mode 100644 index 0000000..5fbb116 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/DataExtractServiceImpl.java @@ -0,0 +1,391 @@ +package org.opengauss.datachecker.extract.service; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.constant.Constants; +import org.opengauss.datachecker.common.entry.enums.DML; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.*; +import org.opengauss.datachecker.common.exception.ProcessMultipleException; +import org.opengauss.datachecker.common.exception.TableNotExistException; +import org.opengauss.datachecker.common.exception.TaskNotFoundException; +import org.opengauss.datachecker.common.util.ThreadUtil; +import org.opengauss.datachecker.extract.cache.MetaDataCache; +import org.opengauss.datachecker.extract.cache.TableExtractStatusCache; +import org.opengauss.datachecker.extract.client.CheckingFeignClient; +import org.opengauss.datachecker.extract.config.ExtractProperties; +import org.opengauss.datachecker.extract.kafka.KafkaAdminService; +import org.opengauss.datachecker.extract.kafka.KafkaCommonService; +import org.opengauss.datachecker.extract.task.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.DependsOn; +import org.springframework.lang.NonNull; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import javax.validation.constraints.NotEmpty; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +@Slf4j +@Service +@DependsOn("extractThreadExecutor") +public class DataExtractServiceImpl implements DataExtractService { + + + /** + * 执行数据抽取任务的线程 最大休眠次数 + */ + private static final int MAX_SLEEP_COUNT = 30; + /** + * 执行数据抽取任务的线程 每次休眠时间，单位毫秒 + */ + private static final int MAX_SLEEP_MILLIS_TIME = 2000; + private static final String PROCESS_NO_RESET = "0"; + + /** + * 服务启动后，会对{code atomicProcessNo}属性进行初始化， + *

+ * 用户启动校验流程，会对{code atomicProcessNo}属性进行校验和设置 + */ + private final AtomicReference atomicProcessNo = new AtomicReference<>(PROCESS_NO_RESET); + + private final AtomicReference> taskReference = new AtomicReference<>(); + private final AtomicReference> incrementTaskReference = new AtomicReference<>(); + + @Autowired + @Qualifier("extractThreadExecutor") + private ThreadPoolTaskExecutor extractThreadExecutor; + + @Autowired + private ExtractTaskBuilder extractTaskBuilder; + + @Autowired + private ExtractThreadSupport extractThreadSupport; + + @Autowired + private IncrementExtractThreadSupport incrementExtractThreadSupport; + + @Autowired + private CheckingFeignClient checkingFeignClient; + + @Autowired + private ExtractProperties extractProperties; + + @Autowired + private KafkaCommonService kafkaCommonService; + + @Autowired + private KafkaAdminService kafkaAdminService; + + @Autowired + private DataManipulationService dataManipulationService; + + /** + * 数据抽取服务 + *

+ * 校验服务通过下发数据抽取流程请求，抽取服务对进程号进行校验，防止同一时间重复发起启动命令 + *

+ * 根据元数据缓存信息，构建数据抽取任务，保存当前任务信息到{@code taskReference}中，等待校验服务发起任务执行指令。 + * 上报任务列表到校验服务。 + * + * @param processNo 执行进程编号 + * @throws ProcessMultipleException 当前实例正在执行数据抽取服务，不能重新开启新的校验。 + */ + @Override + public List buildExtractTaskAllTables(String processNo) throws ProcessMultipleException { + // 调用端点不是源端，则直接返回空 + if (!Objects.equals(extractProperties.getEndpoint(), Endpoint.SOURCE)) { + log.info("The current endpoint is not the source endpoint, and the task cannot be built"); + return Collections.EMPTY_LIST; + } + if (atomicProcessNo.compareAndSet(PROCESS_NO_RESET, processNo)) { + Set tableNames = MetaDataCache.getAllKeys(); + List taskList = extractTaskBuilder.builder(tableNames); + if (CollectionUtils.isEmpty(taskList)) { + return taskList; + } + taskReference.set(taskList); + log.info("build extract task process={} count={}", processNo, taskList.size()); + atomicProcessNo.set(processNo); + + List taskNameList = taskList.stream() + .map(ExtractTask::getTaskName) + .map(String::toUpperCase) + .collect(Collectors.toList()); + initTableExtractStatus(new ArrayList<>(tableNames)); + return taskList; + } else { + log.error("process={} is running extract task , {} please wait ... ", atomicProcessNo.get(), processNo); + throw new ProcessMultipleException("process {" + atomicProcessNo.get() + "} is running extract task"); + } + } + + /** + * 宿端任务配置 + * + * @param processNo 执行进程编号 + * @param taskList 任务列表 + * @throws ProcessMultipleException 前实例正在执行数据抽取服务，不能重新开启新的校验。 + */ + @Override + public void buildExtractTaskAllTables(String processNo, @NonNull List taskList) throws ProcessMultipleException { + if (!Objects.equals(extractProperties.getEndpoint(), Endpoint.SINK)) { + return; + } + // 校验源端构建的任务列表 在宿端是否存在 ，将不存在任务列表过滤 + final Set tableNames = MetaDataCache.getAllKeys(); + if (atomicProcessNo.compareAndSet(PROCESS_NO_RESET, processNo)) { + if (CollectionUtils.isEmpty(taskList) || CollectionUtils.isEmpty(tableNames)) { + return; + } + final List extractTasks = taskList.stream() + .filter(task -> tableNames.contains(task.getTableName())) + .collect(Collectors.toList()); + taskReference.set(extractTasks); + log.info("build extract task process={} count={}", processNo, extractTasks.size()); + atomicProcessNo.set(processNo); + + // taskCountMap用于统计表分片查询的任务数量 + Map taskCountMap = new HashMap<>(Constants.InitialCapacity.MAP); + taskList.forEach(task -> { + if (!taskCountMap.containsKey(task.getTableName())) { + taskCountMap.put(task.getTableName(), task.getDivisionsTotalNumber()); + } + }); + // 初始化数据抽取任务执行状态 + TableExtractStatusCache.init(taskCountMap); + + final List filterTaskTables = taskList.stream() + .filter(task -> !tableNames.contains(task.getTableName())) + .map(ExtractTask::getTableName) + .distinct() + .collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(filterTaskTables)) { + log.info("process={} ,source endpoint database have some tables ,not in the sink tables[{}]", + processNo, filterTaskTables); + } + } else { + log.error("process={} is running extract task , {} please wait ... ", atomicProcessNo.get(), processNo); + throw new ProcessMultipleException("process {" + atomicProcessNo.get() + "} is running extract task"); + } + } + + /** + * 清理当前构建任务 + */ + @Override + public void cleanBuildedTask() { + if (Objects.nonNull(taskReference.getAcquire())) { + taskReference.getAcquire().clear(); + } + if (Objects.nonNull(incrementTaskReference.getAcquire())) { + incrementTaskReference.getAcquire().clear(); + } + TableExtractStatusCache.removeAll(); + atomicProcessNo.set(PROCESS_NO_RESET); + log.info("clear the current build task cache!"); + log.info("clear extraction service status flag!"); + } + + /** + * 查询当前执行流程下，指定表的数据抽取相关信息 + * + * @param tableName 表名称 + * @return 表的数据抽取相关信息 + */ + @Override + public ExtractTask queryTableInfo(String tableName) { + List taskList = taskReference.get(); + Optional taskEntry = Optional.empty(); + if (!CollectionUtils.isEmpty(taskList)) { + for (ExtractTask task : taskList) { + if (Objects.equals(task.getTableName(), tableName)) { + taskEntry = Optional.of(task); + break; + } + } + } + if (taskEntry.isEmpty()) { + throw new TaskNotFoundException(tableName); + } + return taskEntry.get(); + } + + /** + * 执行指定进程编号的数据抽取任务。 + *

+ * 执行抽取任务，对当前进程编号进行校验，并对抽取任务进行校验。 + * 对于抽取任务的校验，采用轮询方式，进行多次校验。 + * 因为源端和宿端的抽取执行逻辑是异步且属于不同的Java进程。为确保不同进程之间流程数据状态一致，采用轮询方式多次进行确认。 + * 若多次确认还不能获取任务数据{@code taskReference}中数据为空，则抛出异常{@link org.opengauss.datachecker.common.exception.TaskNotFoundException} + * + * @param processNo 执行进程编号 + * @throws TaskNotFoundException 任务数据为空，则抛出异常 TaskNotFoundException + */ + @Async + @Override + public void execExtractTaskAllTables(String processNo) throws TaskNotFoundException { + if (Objects.equals(atomicProcessNo.get(), processNo)) { + int sleepCount = 0; + while (CollectionUtils.isEmpty(taskReference.get())) { + ThreadUtil.sleep(MAX_SLEEP_MILLIS_TIME); + if (sleepCount++ > MAX_SLEEP_COUNT) { + log.info("endpoint [{}] and process[{}}] task is empty!", extractProperties.getEndpoint().getDescription(), processNo); + break; + } + } + List taskList = taskReference.get(); + if (CollectionUtils.isEmpty(taskList)) { + return; + } + taskList.forEach(task -> { + log.info("执行数据抽取任务：{}", task); + ThreadUtil.sleep(100); + Topic topic = kafkaCommonService.getTopicInfo(processNo, task.getTableName(), task.getDivisionsTotalNumber()); + kafkaAdminService.createTopic(topic.getTopicName(), topic.getPartitions()); + extractThreadExecutor.submit(new ExtractTaskThread(task, topic, extractThreadSupport)); + }); + } + } + + /** + * 生成修复报告的DML语句 + * + * @param tableName 表名 + * @param dml dml 类型 + * @param diffSet 待生成主键集合 + * @return DML语句 + */ + @Override + public List buildRepairDml(String schema, @NotEmpty String tableName, @NonNull DML dml, @NotEmpty Set diffSet) { + if (CollectionUtils.isEmpty(diffSet)) { + return new ArrayList<>(); + } + List resultList = new ArrayList<>(); + final TableMetadata metadata = MetaDataCache.get(tableName); + final List primaryMetas = metadata.getPrimaryMetas(); + + if (Objects.equals(dml, DML.DELETE)) { + resultList.addAll(dataManipulationService.buildDelete(schema, tableName, diffSet, primaryMetas)); + } else if (Objects.equals(dml, DML.INSERT)) { + resultList.addAll(dataManipulationService.buildInsert(schema, tableName, diffSet, metadata)); + } else if (Objects.equals(dml, DML.REPLACE)) { + resultList.addAll(dataManipulationService.buildReplace(schema, tableName, diffSet, metadata)); + } + return resultList; + } + + /** + * 查询表数据 + * + * @param tableName 表名称 + * @param compositeKeys 复核主键集合 + * @return 主键对应表数据 + */ + @Override + public List> queryTableColumnValues(String tableName, List compositeKeys) { + final TableMetadata metadata = MetaDataCache.get(tableName); + if (Objects.isNull(metadata)) { + throw new TableNotExistException(tableName); + } + return dataManipulationService.queryColumnValues(tableName, new ArrayList<>(compositeKeys), metadata); + } + + /** + * 根据数据变更日志 构建增量抽取任务 + * + * @param sourceDataLogs 数据变更日志 + */ + @Override + public void buildExtractIncrementTaskByLogs(List sourceDataLogs) { + final String schema = extractProperties.getSchema(); + List taskList = extractTaskBuilder.buildIncrementTask(schema, sourceDataLogs); + log.info("构建增量抽取任务完成：{}", taskList.size()); + if (CollectionUtils.isEmpty(taskList)) { + return; + } + incrementTaskReference.set(taskList); + + List tableNameList = sourceDataLogs.stream() + .map(SourceDataLog::getTableName) + .collect(Collectors.toList()); + Map taskCount = new HashMap<>(Constants.InitialCapacity.MAP); + createTaskCountMapping(tableNameList, taskCount); + TableExtractStatusCache.init(taskCount); + initTableExtractStatus(tableNameList); + } + + private void createTaskCountMapping(List tableNameList, Map taskCount) { + tableNameList.forEach(table -> { + taskCount.put(table, 1); + }); + } + + /** + * 执行增量校验数据抽取 + */ + @Override + public void execExtractIncrementTaskByLogs() { + + List taskList = incrementTaskReference.get(); + if (CollectionUtils.isEmpty(taskList)) { + log.info("endpoint [{}] task is empty!", extractProperties.getEndpoint().getDescription()); + return; + } + taskList.forEach(task -> { + log.info("执行数据抽取任务：{}", task); + ThreadUtil.sleep(100); + Topic topic = kafkaCommonService.getIncrementTopicInfo(task.getTableName()); + kafkaAdminService.createTopic(topic.getTopicName(), topic.getPartitions()); + extractThreadExecutor.submit(new IncrementExtractTaskThread(task, topic, incrementExtractThreadSupport)); + }); + } + + /** + * 查询当前表结构元数据信息，并进行Hash + * + * @param tableName 表名称 + * @return 表结构Hash + */ + @Override + public TableMetadataHash queryTableMetadataHash(String tableName) { + return dataManipulationService.queryTableMetadataHash(tableName); + } + + /** + * 查询表指定PK列表数据，并进行Hash 用于二次校验数据查询 + * + * @param dataLog 数据日志 + * @return rowdata hash + */ + @Override + public List querySecondaryCheckRowData(SourceDataLog dataLog) { + final String tableName = dataLog.getTableName(); + final List compositeKeys = dataLog.getCompositePrimaryValues(); + + final TableMetadata metadata = MetaDataCache.get(tableName); + if (Objects.isNull(metadata)) { + throw new TableNotExistException(tableName); + } + List> dataRowList = dataManipulationService.queryColumnValues(tableName, compositeKeys, metadata); + RowDataHashHandler handler = new RowDataHashHandler(); + return handler.handlerQueryResult(metadata, dataRowList); + } + + @Override + public String queryDatabaseSchema() { + return extractProperties.getSchema(); + } + + + private void initTableExtractStatus(List tableNameList) { + if (Objects.equals(extractProperties.getEndpoint(), Endpoint.SOURCE)) { + checkingFeignClient.initTableExtractStatus(new ArrayList<>(tableNameList)); + log.info("通知校验服务初始化增量抽取任务状态：{}", tableNameList); + } + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java new file mode 100644 index 0000000..7b47a09 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/service/MetaDataService.java @@ -0,0 +1,113 @@ +package org.opengauss.datachecker.extract.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.opengauss.datachecker.common.entry.enums.CheckBlackWhiteMode; +import org.opengauss.datachecker.common.entry.enums.ColumnKey; +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.opengauss.datachecker.extract.cache.MetaDataCache; +import org.opengauss.datachecker.extract.dao.DataBaseMetaDataDAOImpl; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import javax.annotation.PostConstruct; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * @author wang chao + * @description 元数据服务 + * @date 2022/5/8 19:27 + * @since 11 + **/ +@Service +@Slf4j +@RequiredArgsConstructor +public class MetaDataService { + + private final DataBaseMetaDataDAOImpl dataBaseMetadataDAOImpl; + + @Value("${spring.extract.query-table-row-count}") + private boolean queryTableRowCount; + + @PostConstruct + public void init() { + MetaDataCache.removeAll(); + Map metaDataMap = queryMetaDataOfSchema(); + MetaDataCache.initCache(); + MetaDataCache.putMap(metaDataMap); + } + + public Map queryMetaDataOfSchema() { + + List tableMetadata = queryTableMetadata(); + List tableNames = tableMetadata + .stream() + .map(TableMetadata::getTableName) + .collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(tableNames)) { + List columnsMetadata = dataBaseMetadataDAOImpl.queryColumnMetadata(tableNames); + Map> tableColumnMap = columnsMetadata.stream().collect(Collectors.groupingBy(ColumnsMetaData::getTableName)); + + tableMetadata.stream().forEach(tableMeta -> { + tableMeta.setColumnsMetas(tableColumnMap.get(tableMeta.getTableName())) + .setPrimaryMetas(getTablePrimaryColumn(tableColumnMap.get(tableMeta.getTableName()))); + }); + log.info("查询数据库元数据信息完成 total=" + columnsMetadata.size()); + } + return tableMetadata.stream().collect(Collectors.toMap(TableMetadata::getTableName, Function.identity())); + } + + public void refushBlackWhiteList(CheckBlackWhiteMode mode, List tableList) { + dataBaseMetadataDAOImpl.resetBlackWhite(mode, tableList); + init(); + } + + public TableMetadata queryMetaDataOfSchema(String tableName) { + TableMetadata tableMetadata = queryTableMetadataByTableName(tableName); + List columnsMetadata = dataBaseMetadataDAOImpl.queryColumnMetadata(List.of(tableName)); + tableMetadata + .setColumnsMetas(columnsMetadata) + .setPrimaryMetas(getTablePrimaryColumn(columnsMetadata)); + + log.info("查询数据库元数据信息完成 total={}", columnsMetadata); + return tableMetadata; + } + + public List queryTableColumnMetaDataOfSchema(String tableName) { + return dataBaseMetadataDAOImpl.queryColumnMetadata(List.of(tableName)); + } + + private TableMetadata queryTableMetadataByTableName(String tableName) { + final List tableMetadatas = queryTableMetadata(); + return tableMetadatas.stream() + .filter(meta -> StringUtils.equalsIgnoreCase(meta.getTableName(), tableName)) + .findFirst() + .orElseGet(null); + } + + private List queryTableMetadata() { + if (queryTableRowCount) { + return dataBaseMetadataDAOImpl.queryTableMetadata(); + } else { + return dataBaseMetadataDAOImpl.queryTableMetadataFast(); + } + } + + /** + * 获取表主键列元数据信息 + * + * @param columnsMetaData + * @return + */ + private List getTablePrimaryColumn(List columnsMetaData) { + return columnsMetaData.stream() + .filter(meta -> ColumnKey.PRI.equals(meta.getColumnKey())) + .collect(Collectors.toList()); + } + +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/DataManipulationService.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/DataManipulationService.java new file mode 100644 index 0000000..9fd805b --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/DataManipulationService.java @@ -0,0 +1,290 @@ +package org.opengauss.datachecker.extract.task; + +import org.apache.commons.lang3.StringUtils; +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.opengauss.datachecker.common.entry.extract.TableMetadataHash; +import org.opengauss.datachecker.common.util.HashUtil; +import org.opengauss.datachecker.extract.config.ExtractProperties; +import org.opengauss.datachecker.extract.constants.ExtConstants; +import org.opengauss.datachecker.extract.dml.*; +import org.opengauss.datachecker.extract.service.MetaDataService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +import java.sql.ResultSetMetaData; +import java.util.*; +import java.util.stream.Collectors; + +/** + * DML 数据操作服务 实现数据的动态查询 + * + * @author ：wangchao + * @date ：Created in 2022/6/13 + * @since ：11 + */ +@Service +public class DataManipulationService { + + @Autowired + private JdbcTemplate jdbcTemplateOne; + @Autowired + private MetaDataService metaDataService; + @Autowired + private ExtractProperties extractProperties; + + public List> queryColumnValues(String tableName, List compositeKeys, TableMetadata metadata) { + Assert.isTrue(Objects.nonNull(metadata), "表元数据信息异常，构建Select SQL失败"); + final List primaryMetas = metadata.getPrimaryMetas(); + + Assert.isTrue(!CollectionUtils.isEmpty(primaryMetas), "表主键元数据信息异常，构建Select SQL失败"); + + // 单一主键表数据查询 + if (primaryMetas.size() == 1) { + final ColumnsMetaData primaryData = primaryMetas.get(0); + String querySql = new SelectDmlBuilder() + .schema(extractProperties.getSchema()) + .columns(metadata.getColumnsMetas()) + .tableName(tableName) + .conditionPrimary(primaryData) + .build(); + return queryColumnValues(querySql, compositeKeys); + } else { + // 复合主键表数据查询 + final SelectDmlBuilder dmlBuilder = new SelectDmlBuilder(); + String querySql = dmlBuilder + .schema(extractProperties.getSchema()) + .columns(metadata.getColumnsMetas()) + .tableName(tableName) + .conditionCompositePrimary(primaryMetas) + .build(); + List batchParam = dmlBuilder.conditionCompositePrimaryValue(primaryMetas, compositeKeys); + return queryColumnValuesByCompositePrimary(querySql, batchParam); + } + } + + /** + * 复合主键表数据查询 + * + * @param selectDml 查询SQL + * @param batchParam 复合主键查询参数 + * @return 查询数据结果 + */ + private List> queryColumnValuesByCompositePrimary(String selectDml, List batchParam) { + // 查询当前任务数据，并对数据进行规整 + HashMap paramMap = new HashMap<>(); + paramMap.put(DmlBuilder.PRIMARY_KEYS, batchParam); + + return queryColumnValues(selectDml, paramMap); + } + + /** + * 单一主键表数据查询 + * + * @param selectDml 查询SQL + * @param primaryKeys 查询主键集合 + * @return 查询数据结果 + */ + private List> queryColumnValues(String selectDml, List primaryKeys) { + // 查询当前任务数据，并对数据进行规整 + HashMap paramMap = new HashMap<>(); + paramMap.put(DmlBuilder.PRIMARY_KEYS, primaryKeys); + + return queryColumnValues(selectDml, paramMap); + } + + /** + * 主键表数据查询 + * + * @param selectDml 查询SQL + * @param paramMap 查询参数 + * @return 查询结果 + */ + private List> queryColumnValues(String selectDml, Map paramMap) { + // 使用JDBC查询当前任务抽取数据 + NamedParameterJdbcTemplate jdbc = new NamedParameterJdbcTemplate(jdbcTemplateOne); + return jdbc.query(selectDml, paramMap, (rs, rowNum) -> { + // 获取当前结果集对应的元数据信息 + ResultSetMetaData metaData = rs.getMetaData(); + // 结果集处理器 + ResultSetHandler handler = new ResultSetHandler(); + // 查询结果集 根据元数据信息 进行数据转换 + return handler.putOneResultSetToMap(rs, metaData); + }); + } + + /** + * 构建指定表的 Replace SQL语句 + * + * @param tableName 表名称 + * @param compositeKeySet 复合主键集合 + * @param metadata 元数据信息 + * @return 返回SQL列表 + */ + public List buildReplace(String schema, String tableName, Set compositeKeySet, TableMetadata metadata) { + List resultList = new ArrayList<>(); + final String localSchema = getLocalSchema(schema); + ReplaceDmlBuilder builder = new ReplaceDmlBuilder().schema(localSchema) + .tableName(tableName) + .columns(metadata.getColumnsMetas()); + + List> columnValues = queryColumnValues(tableName, new ArrayList<>(compositeKeySet), metadata); + Map> compositeKeyValues = transtlateColumnValues(columnValues, metadata.getPrimaryMetas()); + compositeKeySet.forEach(compositeKey -> { + Map columnValue = compositeKeyValues.get(compositeKey); + if (Objects.nonNull(columnValue) && !columnValue.isEmpty()) { + resultList.add(builder.columnsValue(columnValue, metadata.getColumnsMetas()).build()); + } + }); + return resultList; + } + + + /** + * 构建指定表的 Insert SQL语句 + * + * @param tableName 表名称 + * @param compositeKeySet 复合主键集合 + * @param metadata 元数据信息 + * @return 返回SQL列表 + */ + public List buildInsert(String schema, String tableName, Set compositeKeySet, TableMetadata metadata) { + + List resultList = new ArrayList<>(); + final String localSchema = getLocalSchema(schema); + InsertDmlBuilder builder = new InsertDmlBuilder().schema(localSchema) + .tableName(tableName) + .columns(metadata.getColumnsMetas()); + + List> columnValues = queryColumnValues(tableName, new ArrayList<>(compositeKeySet), metadata); + Map> compositeKeyValues = transtlateColumnValues(columnValues, metadata.getPrimaryMetas()); + compositeKeySet.forEach(compositeKey -> { + Map columnValue = compositeKeyValues.get(compositeKey); + if (Objects.nonNull(columnValue) && !columnValue.isEmpty()) { + resultList.add(builder.columnsValue(columnValue, metadata.getColumnsMetas()).build()); + } + }); + return resultList; + } + + private Map> transtlateColumnValues(List> columnValues, List primaryMetas) { + final List primaryKeys = getCompositeKeyColumns(primaryMetas); + Map> map = new HashMap<>(); + columnValues.forEach(values -> { + map.put(getCompositeKey(values, primaryKeys), values); + }); + return map; + } + + private List getCompositeKeyColumns(List primaryMetas) { + return primaryMetas.stream().map(ColumnsMetaData::getColumnName).collect(Collectors.toUnmodifiableList()); + } + + private String getCompositeKey(Map columnValues, List primaryKeys) { + return primaryKeys.stream().map(key -> columnValues.get(key)).collect(Collectors.joining(ExtConstants.PRIMARY_DELIMITER)); + } + + + /** + * 构建指定表的批量 Delete SQL语句 + * + * @param tableName 表名称 + * @param compositeKeySet 复合主键集合 + * @param primaryMetas 主键元数据信息 + * @return 返回SQL列表 + */ + public List buildBatchDelete(String schema, String tableName, Set compositeKeySet, List primaryMetas) { + List resultList = new ArrayList<>(); + final String localSchema = getLocalSchema(schema); + if (primaryMetas.size() == 1) { + final ColumnsMetaData primaryMeta = primaryMetas.stream().findFirst().get(); + compositeKeySet.forEach(compositeKey -> { + final String deleteDml = new BatchDeleteDmlBuilder() + .tableName(tableName) + .schema(localSchema) + .conditionPrimary(primaryMeta) + .build(); + resultList.add(deleteDml); + }); + } else { + compositeKeySet.forEach(compositeKey -> { + resultList.add(new BatchDeleteDmlBuilder() + .tableName(tableName) + .schema(localSchema) + .conditionCompositePrimary(primaryMetas) + .build()); + }); + } + + return resultList; + } + + /** + * 构建指定表的 Delete SQL语句 + * + * @param tableName 表名称 + * @param compositeKeySet 复合主键集合 + * @param primaryMetas 主键元数据信息 + * @return 返回SQL列表 + */ + public List buildDelete(String schema, String tableName, Set compositeKeySet, List primaryMetas) { + + List resultList = new ArrayList<>(); + final String localSchema = getLocalSchema(schema); + if (primaryMetas.size() == 1) { + final ColumnsMetaData primaryMeta = primaryMetas.stream().findFirst().get(); + compositeKeySet.forEach(compositeKey -> { + final String deleteDml = new DeleteDmlBuilder() + .tableName(tableName) + .schema(localSchema) + .condition(primaryMeta, compositeKey) + .build(); + resultList.add(deleteDml); + }); + } else { + compositeKeySet.forEach(compositeKey -> { + resultList.add(new DeleteDmlBuilder() + .tableName(tableName) + .schema(localSchema) + .conditionCompositePrimary(compositeKey, primaryMetas) + .build()); + }); + } + + return resultList; + } + + private String getLocalSchema(String schema) { + if (StringUtils.isEmpty(schema)) { + return extractProperties.getSchema(); + } + return schema; + } + + /** + * 查询当前表结构元数据信息，并进行Hash + * + * @param tableName 表名称 + * @return 表结构Hash + */ + public TableMetadataHash queryTableMetadataHash(String tableName) { + final TableMetadataHash tableMetadataHash = new TableMetadataHash().setTableName(tableName); + final List columnsMetaData = metaDataService.queryTableColumnMetaDataOfSchema(tableName); + StringBuffer buffer = new StringBuffer(); + if (!CollectionUtils.isEmpty(columnsMetaData)) { + columnsMetaData.sort(Comparator.comparing(ColumnsMetaData::getColumnName)); + columnsMetaData.forEach(column -> { + buffer.append(column.getColumnName()) + .append(column.getColumnType()) + .append(column.getDataType()) + .append(column.getOrdinalPosition()); + }); + } + tableMetadataHash.setTableHash(HashUtil.hashBytes(buffer.toString())); + return tableMetadataHash; + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractTaskBuilder.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractTaskBuilder.java new file mode 100644 index 0000000..37d5957 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractTaskBuilder.java @@ -0,0 +1,188 @@ +package org.opengauss.datachecker.extract.task; + +import org.opengauss.datachecker.common.entry.extract.ExtractIncrementTask; +import org.opengauss.datachecker.common.entry.extract.ExtractTask; +import org.opengauss.datachecker.common.entry.extract.SourceDataLog; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.opengauss.datachecker.extract.cache.MetaDataCache; +import org.opengauss.datachecker.extract.cache.TableExtractStatusCache; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * @author wang chao + * @description 数据抽取任务构建器 + * @date 2022/5/8 19:27 + * @since 11 + **/ +@Service +public class ExtractTaskBuilder { + private static final int EXTRACT_MAX_ROW_COUNT = 100000; + private static final String TASK_NAME_PREFIX = "TASK_TABLE_"; + private static final String INCREMENT_TASK_NAME_PREFIX = "INCREMENT_TASK_TABLE_"; + + + /** + * 

+     * 根据元数据缓存信息构建 表数据抽取任务。并初始化数据抽取任务执行状态。
+     * 任务构建依赖于元数据缓存信息，以及元数据缓存中加载的当前表记录总数。单个分片任务查询数据总数不超过{@value EXTRACT_MAX_ROW_COUNT}
+     * {@code taskCountMap} 用于统计所有待抽取表的分片查询的任务数量
+     * {@code tableRows} 为表元数据信息中统计的当前表数据量
+     *
+ * + * @param tableNames 待构建抽取任务表集合 + * @return 任务列表 + */ + public List builder(Set tableNames) { + Assert.isTrue(!CollectionUtils.isEmpty(tableNames), "构建数据抽取任务表不能为空"); + List taskList = new ArrayList<>(); + + final List tableNameOrderList = tableNames.stream().sorted((tableName1, tableName2) -> { + TableMetadata metadata1 = MetaDataCache.get(tableName1); + TableMetadata metadata2 = MetaDataCache.get(tableName2); + // 排序异常情况处理 + if (Objects.isNull(metadata1) && Objects.isNull(metadata2)) { + return 0; + } + if (Objects.isNull(metadata1)) { + return -1; + } + if (Objects.isNull(metadata2)) { + return 1; + } + return (int) (metadata1.getTableRows() - metadata2.getTableRows()); + }).collect(Collectors.toList()); + // taskCountMap用于统计表分片查询的任务数量 + Map taskCountMap = new HashMap<>(); + tableNameOrderList.forEach(tableName -> { + TableMetadata metadata = MetaDataCache.get(tableName); + if (Objects.nonNull(metadata)) { + // tableRows为表元数据信息中统计的当前表数据量 + long tableRows = metadata.getTableRows(); + if (tableRows > EXTRACT_MAX_ROW_COUNT) { + + // 根据表元数据信息构建抽取任务 + List taskEntryList = buildTaskList(metadata); + taskCountMap.put(tableName, taskEntryList.size()); + taskList.addAll(taskEntryList); + } else { + taskList.add(buildTask(metadata)); + taskCountMap.put(tableName, 1); + } + } + }); + + // 初始化数据抽取任务执行状态 + TableExtractStatusCache.init(taskCountMap); + return taskList; + } + + private ExtractTask buildTask(TableMetadata metadata) { + return new ExtractTask().setDivisionsTotalNumber(1) + .setTableMetadata(metadata) + .setDivisionsTotalNumber(1) + .setDivisionsOrdinal(1) + .setOffset(metadata.getTableRows()) + .setStart(0) + .setTableName(metadata.getTableName()) + .setTaskName(taskNameBuilder(metadata.getTableName(), 1, 1)); + } + + + /** + * 根据表元数据信息 构建表数据抽取任务。 + * 根据元数据信息中表数据总数估值进行任务分片，单个分片任务查询数据总数不超过 {@value EXTRACT_MAX_ROW_COUNT} + * + * @param metadata 元数据信息 + * @return 任务列表 + */ + private List buildTaskList(TableMetadata metadata) { + List taskList = new ArrayList<>(); + long tableRows = metadata.getTableRows(); + final int taskCount = calcTaskCount(tableRows); + + IntStream.rangeClosed(1, taskCount).forEach(idx -> { + long remainingExtractNumber = tableRows - (idx - 1) * EXTRACT_MAX_ROW_COUNT; + ExtractTask extractTask = buildExtractTask(taskCount, idx, EXTRACT_MAX_ROW_COUNT, remainingExtractNumber); + extractTask.setDivisionsTotalNumber(taskCount) + .setTableMetadata(metadata) + .setTableName(metadata.getTableName()) + .setTaskName(taskNameBuilder(metadata.getTableName(), taskCount, idx)); + taskList.add(extractTask); + }); + return taskList; + } + + /** + * 根据表记录总数，计算分片任务数量 + * + * @param tableRows 表记录总数 + * @return 分拆任务总数 + */ + private int calcTaskCount(long tableRows) { + return (int) (tableRows / EXTRACT_MAX_ROW_COUNT); + } + + /** + * 任务名称构建 + * 
+     * 若任务分拆总数大于1，名称由：前缀信息 {@value TASK_NAME_PREFIX} 、表名称 、表序列 构建
+     * 若任务分拆总数为1，即未拆分 ，则根据 前缀信息 {@value TASK_NAME_PREFIX} 、表名称 构建
+     *
+ * + * @param tableName 表名 + * @param taskCount 任务分拆总数 + * @param ordinal 表任务分拆序列 + * @return 任务名称 + */ + private String taskNameBuilder(@NonNull String tableName, int taskCount, int ordinal) { + if (taskCount > 1) { + return TASK_NAME_PREFIX.concat(tableName.toUpperCase()).concat("_").concat(String.valueOf(ordinal)); + } else { + return TASK_NAME_PREFIX.concat(tableName.toUpperCase()); + } + } + + /** + * @param taskCount 任务总数 + * @param ordinal 任务序列 + * @param planedExtractNumber 当前任务计划抽取记录总数 + * @param remainingExtractNumber 实际剩余抽取记录总数 + * @return 构建任务对象 + */ + private ExtractTask buildExtractTask(int taskCount, int ordinal, long planedExtractNumber, long remainingExtractNumber) { + ExtractTask extractTask = new ExtractTask() + .setDivisionsOrdinal(ordinal) + .setStart(((ordinal - 1) * planedExtractNumber)) + .setOffset(ordinal == taskCount ? remainingExtractNumber : planedExtractNumber); + return extractTask; + } + + /** + * 增量任务构建 + * + * @param schema schema + * @param sourceDataLogs 增量日志 + * @return 增量任务 + */ + public List buildIncrementTask(String schema, List sourceDataLogs) { + List incrementTasks = new ArrayList<>(); + sourceDataLogs.forEach(datalog -> { + incrementTasks.add(new ExtractIncrementTask().setSchema(schema) + .setSourceDataLog(datalog) + .setTableName(datalog.getTableName()) + .setTaskName(incrementTaskNameBuilder(datalog.getTableName()))); + }); + return incrementTasks; + } + + private String incrementTaskNameBuilder(@NonNull String tableName) { + return INCREMENT_TASK_NAME_PREFIX.concat(tableName.toUpperCase()); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractTaskThread.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractTaskThread.java new file mode 100644 index 0000000..e906a11 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractTaskThread.java @@ -0,0 +1,126 @@ +package org.opengauss.datachecker.extract.task; + + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.*; +import org.opengauss.datachecker.common.util.ThreadUtil; +import org.opengauss.datachecker.extract.cache.TableExtractStatusCache; +import org.opengauss.datachecker.extract.client.CheckingFeignClient; +import org.opengauss.datachecker.extract.kafka.KafkaProducerService; +import org.opengauss.datachecker.extract.util.HashHandler; +import org.opengauss.datachecker.extract.util.MetaDataUtil; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.util.CollectionUtils; + +import java.sql.*; +import java.util.*; + +/** + * @author wang chao + * @description 数据抽取线程类 + * @date 2022/5/12 19:17 + * @since 11 + **/ +@Slf4j +public class ExtractTaskThread implements Runnable { + + /** + * 数据推送Kafka Topic信息 + */ + private final Topic topic; + /** + * 当前抽取任务对象 + */ + private final ExtractTask task; + /** + * 当前执行端点信息 + */ + private final Endpoint endpoint; + private final String schema; + + private final JdbcTemplate jdbcTemplate; + private final KafkaProducerService kafkaProducerService; + private final CheckingFeignClient checkingFeignClient; + + + /** + * 线程构造函数 + * + * @param task Kafka Topic信息 + * @param topic 数据抽取流程编号 + * @param support 线程参数封装 + */ + public ExtractTaskThread(ExtractTask task, Topic topic, ExtractThreadSupport support) { + this.task = task; + this.topic = topic; + this.schema = support.getExtractProperties().getSchema(); + this.endpoint = support.getExtractProperties().getEndpoint(); + this.jdbcTemplate = new JdbcTemplate(support.getDataSourceOne()); + this.kafkaProducerService = support.getKafkaProducerService(); + this.checkingFeignClient = support.getCheckingFeignClient(); + } + + + @Override + public void run() { + log.info("start extract task={}", task.getTaskName()); + + TableMetadata tableMetadata = task.getTableMetadata(); + + // 根据当前任务中表元数据信息，构造查询SQL + String sql = new SelectSqlBulder(tableMetadata, schema, task.getStart(), task.getOffset()).builder(); + // 通过JDBC SQL 查询数据 + List> dataRowList = queryColumnValues(sql); + log.info("query extract task={} completed", task.getTaskName()); + // 对查询出的数据结果 进行哈希计算 + RowDataHashHandler handler = new RowDataHashHandler(); + List recordHashList = handler.handlerQueryResult(tableMetadata, dataRowList); + log.info("hash extract task={} completed", task.getTaskName()); + // 推送本地缓存 根据分片顺序将数据推送到kafka + String tableName = task.getTableName(); + // 当前分片任务，之前的任务状态未执行完成，请稍后再次检查尝试 + kafkaProducerService.syncSend(topic, recordHashList); + log.info("send kafka extract task={} completed", task.getTaskName()); + while (task.isDivisions() && !TableExtractStatusCache.checkComplated(tableName, task.getDivisionsOrdinal())) { + log.debug("task=[{}] wait divisions of before , send data to kafka completed", task.getTaskName()); + ThreadUtil.sleep(100); + } + // 推送完成则更新当前任务的抽取状态 + TableExtractStatusCache.update(tableName, task.getDivisionsOrdinal()); + log.info("update extract task={} status completed", task.getTaskName()); + if (!task.isDivisions()) { + // 通知校验服务，当前表对应任务数据抽取已经完成 + checkingFeignClient.refushTableExtractStatus(tableName, endpoint); + log.info("refush table extract status tableName={} status completed", task.getTaskName()); + } + if (task.isDivisions() && task.getDivisionsOrdinal() == task.getDivisionsTotalNumber()) { + // 当前表的数据抽取任务完成(所有子任务均完成) + // 通知校验服务，当前表对应任务数据抽取已经完成 + checkingFeignClient.refushTableExtractStatus(tableName, endpoint); + log.info("refush table=[{}] extract status completed,task=[{}]", tableName, task.getTaskName()); + } + } + + /** + * 通过JDBC SQL 查询数据 + * + * @param sql 执行SQL + * @return 查询结果 + */ + private List> queryColumnValues(String sql) { + Map map = new HashMap<>(); + // 使用JDBC查询当前任务抽取数据 + NamedParameterJdbcTemplate jdbc = new NamedParameterJdbcTemplate(jdbcTemplate); + // 查询当前任务数据，并对数据进行规整 + return jdbc.query(sql, map, (rs, rowNum) -> { + // 获取当前结果集对应的元数据信息 + ResultSetMetaData metaData = rs.getMetaData(); + // 结果集处理器 + ResultSetHandler handler = new ResultSetHandler(); + // 查询结果集 根据元数据信息 进行数据转换 + return handler.putOneResultSetToMap(rs, metaData); + }); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractThreadSupport.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractThreadSupport.java new file mode 100644 index 0000000..1de539a --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ExtractThreadSupport.java @@ -0,0 +1,33 @@ +package org.opengauss.datachecker.extract.task; + +import lombok.Getter; +import org.opengauss.datachecker.extract.client.CheckingFeignClient; +import org.opengauss.datachecker.extract.config.ExtractProperties; +import org.opengauss.datachecker.extract.kafka.KafkaProducerService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.sql.DataSource; + +/** + * 抽取线程 参数封装 + * + * @author ：wangchao + * @date ：Created in 2022/5/30 + * @since ：11 + */ +@Getter +@Service +public class ExtractThreadSupport { + @Autowired + private DataSource dataSourceOne; + + @Autowired + private KafkaProducerService kafkaProducerService; + + @Autowired + private CheckingFeignClient checkingFeignClient; + + @Autowired + private ExtractProperties extractProperties; +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/IncrementExtractTaskThread.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/IncrementExtractTaskThread.java new file mode 100644 index 0000000..7ea669c --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/IncrementExtractTaskThread.java @@ -0,0 +1,191 @@ +package org.opengauss.datachecker.extract.task; + + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.entry.enums.Endpoint; +import org.opengauss.datachecker.common.entry.extract.*; +import org.opengauss.datachecker.common.exception.ExtractException; +import org.opengauss.datachecker.extract.cache.TableExtractStatusCache; +import org.opengauss.datachecker.extract.client.CheckingFeignClient; +import org.opengauss.datachecker.extract.dml.DmlBuilder; +import org.opengauss.datachecker.extract.dml.SelectDmlBuilder; +import org.opengauss.datachecker.extract.kafka.KafkaProducerService; +import org.opengauss.datachecker.extract.service.MetaDataService; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.util.CollectionUtils; + +import java.sql.ResultSetMetaData; +import java.util.*; + +/** + * @author wang chao + * @description 数据抽取线程类 + * @date 2022/5/12 19:17 + * @since 11 + **/ +@Slf4j +public class IncrementExtractTaskThread implements Runnable { + + /** + * SQL 单次查询语句，构建查询参数最大个数 + */ + private static final int MAX_QUERY_ROW_COUNT = 1000; + + private final Topic topic; + private final String schema; + private final String taskName; + private final String tableName; + private final Endpoint endpoint; + private final SourceDataLog sourceDataLog; + private final JdbcTemplate jdbcTemplate; + private final KafkaProducerService kafkaProducerService; + private final CheckingFeignClient checkingFeignClient; + private final MetaDataService metaDataService; + + private boolean singlePrimaryKey; + + /** + * 线程构造函数 + * + * @param task Kafka Topic信息 + * @param topic 数据抽取流程编号 + * @param support 线程参数封装 + */ + public IncrementExtractTaskThread(ExtractIncrementTask task, Topic topic, IncrementExtractThreadSupport support) { + this.topic = topic; + this.schema = support.getExtractProperties().getSchema(); + this.endpoint = support.getExtractProperties().getEndpoint(); + this.tableName = task.getTableName(); + this.taskName = task.getTaskName(); + this.sourceDataLog = task.getSourceDataLog(); + this.jdbcTemplate = new JdbcTemplate(support.getDataSourceOne()); + this.kafkaProducerService = support.getKafkaProducerService(); + this.checkingFeignClient = support.getCheckingFeignClient(); + this.metaDataService = support.getMetaDataService(); + } + + + @Override + public void run() { + log.info("start extract task={}", taskName); + TableMetadata tableMetadata = getTableMetadata(); + + // 根据当前任务中表元数据信息，构造查询SQL + SelectDmlBuilder sqlBuilder = buildSelectSql(tableMetadata, schema); + + // 查询当前任务数据，并对数据进行规整 + HashMap paramMap = new HashMap<>(); + final List compositePrimaryValues = sourceDataLog.getCompositePrimaryValues(); + paramMap.put(DmlBuilder.PRIMARY_KEYS, getSqlParam(sqlBuilder, tableMetadata.getPrimaryMetas(), compositePrimaryValues)); + + // 查询当前任务数据，并对数据进行规整 + List> dataRowList = queryColumnValues(sqlBuilder.build(), paramMap); + log.info("query extract task={} completed row count=[{}]", taskName, dataRowList.size()); + // 对查询出的数据结果 进行哈希计算 + RowDataHashHandler handler = new RowDataHashHandler(); + List recordHashList = handler.handlerQueryResult(tableMetadata, dataRowList); + log.info("hash extract task={} completed", taskName); + // 推送本地缓存 根据分片顺序将数据推送到kafka + kafkaProducerService.syncSend(topic, recordHashList); + log.info("send kafka extract task={} completed", taskName); + // 推送完成则更新当前任务的抽取状态 + TableExtractStatusCache.update(tableName, 1); + log.info("update extract task={} status completed", tableName); + // 通知校验服务，当前表对应任务数据抽取已经完成 + checkingFeignClient.refushTableExtractStatus(tableName, endpoint); + log.info("refush table extract status tableName={} status completed", tableName); + + } + + /** + * 查询SQL构建后期优化 + * 查询SQL 构建 select colums from table where pk in(...)

+ * 后期优化方式：

+ * 单主键方式 + * SELECT * + * FROM ( + * SELECT '14225351881572354' cid UNION ALL + * SELECT '14225351898349591' UNION ALL + * SELECT '14225351902543878' + * ) AS tmp, test.test1 t + * WHERE tmp.cid = t.b_number;

+ *

+ * 复合主键方式 + * SELECT * + * FROM ( + * SELECT '1523567590573785088' cid,'type_01' ctype UNION ALL + * SELECT '1523567590573785188','type_01' UNION ALL + * SELECT '1523567590573785189','type_03' + * ) AS tmp, test.test2 t + * WHERE tmp.cid = t.b_number AND tmp.ctype=t.b_type; + * + * @param tableMetadata 表元数据信息 + * @param schema 数据库schema + * @return SQL构建器对象 + */ + private SelectDmlBuilder buildSelectSql(TableMetadata tableMetadata, String schema) { + // 复合主键表数据查询 + SelectDmlBuilder dmlBuilder = new SelectDmlBuilder(); + final List primaryMetas = tableMetadata.getPrimaryMetas(); + if (singlePrimaryKey) { + final ColumnsMetaData primaryData = primaryMetas.get(0); + dmlBuilder.schema(schema) + .columns(tableMetadata.getColumnsMetas()) + .tableName(tableMetadata.getTableName()) + .conditionPrimary(primaryData); + } else { + // 复合主键表数据查询 + dmlBuilder.schema(schema) + .columns(tableMetadata.getColumnsMetas()) + .tableName(tableMetadata.getTableName()) + .conditionCompositePrimary(primaryMetas); + } + return dmlBuilder; + } + + /** + * 构建JDBC 查询参数 + * + * @param sqlBuilder SQL构建器 + * @param primaryMetas 主键信息 + * @param compositePrimaryValues 查询参数 + * @return 封装后的JDBC查询参数 + */ + private List getSqlParam(SelectDmlBuilder sqlBuilder, List primaryMetas, List compositePrimaryValues) { + if (singlePrimaryKey) { + return compositePrimaryValues; + } else { + return sqlBuilder.conditionCompositePrimaryValue(primaryMetas, compositePrimaryValues); + } + } + + /** + * 主键表数据查询 + * + * @param selectDml 查询SQL + * @param paramMap 查询参数 + * @return 查询结果 + */ + private List> queryColumnValues(String selectDml, Map paramMap) { + // 使用JDBC查询当前任务抽取数据 + NamedParameterJdbcTemplate jdbc = new NamedParameterJdbcTemplate(jdbcTemplate); + return jdbc.query(selectDml, paramMap, (rs, rowNum) -> { + // 获取当前结果集对应的元数据信息 + ResultSetMetaData metaData = rs.getMetaData(); + // 结果集处理器 + ResultSetHandler handler = new ResultSetHandler(); + // 查询结果集 根据元数据信息 进行数据转换 + return handler.putOneResultSetToMap(rs, metaData); + }); + } + + private TableMetadata getTableMetadata() { + final TableMetadata metadata = metaDataService.queryMetaDataOfSchema(tableName); + if (Objects.isNull(metadata) || CollectionUtils.isEmpty(metadata.getPrimaryMetas())) { + throw new ExtractException(tableName + " metadata not found！"); + } + this.singlePrimaryKey = metadata.getPrimaryMetas().size() == 1; + return metadata; + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/IncrementExtractThreadSupport.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/IncrementExtractThreadSupport.java new file mode 100644 index 0000000..26b3c59 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/IncrementExtractThreadSupport.java @@ -0,0 +1,21 @@ +package org.opengauss.datachecker.extract.task; + +import lombok.Getter; +import org.opengauss.datachecker.extract.service.MetaDataService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 抽取线程 参数封装 + * + * @author ：wangchao + * @date ：Created in 2022/5/30 + * @since ：11 + */ +@Getter +@Service +public class IncrementExtractThreadSupport extends ExtractThreadSupport { + + @Autowired + private MetaDataService metaDataService; +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ResultSetHandler.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ResultSetHandler.java new file mode 100644 index 0000000..e5cbccf --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/ResultSetHandler.java @@ -0,0 +1,162 @@ +package org.opengauss.datachecker.extract.task; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/13 + * @since ：11 + */ + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.lang.NonNull; + +import java.sql.*; +import java.sql.Date; +import java.text.SimpleDateFormat; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.IntStream; + +/** + * @author wang chao + * @description 结果集对象处理器 + * @since 11 + **/ +@Slf4j +public class ResultSetHandler { + + private final ObjectMapper mapper = ObjectMapperWapper.getObjectMapper(); + private static final List SQL_TIME_TYPES = List.of(Types.DATE, Types.TIME, Types.TIMESTAMP, Types.TIME_WITH_TIMEZONE, Types.TIMESTAMP_WITH_TIMEZONE); + + private static final DateTimeFormatter DATE = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("HH:mm:ss"); + private static final DateTimeFormatter TIMESTAMP = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); + + /** + * 将当前查询结果集 根据结果集元数据信息转换为Map + * + * @param resultSet JDBC 数据查询结果集 + * @param rsmd JDBC 结果集元数据 + * @return JDBC 数据封装结果 + * @throws SQLException 返回SQL异常 + */ + public Map putOneResultSetToMap(ResultSet resultSet, ResultSetMetaData rsmd) throws SQLException { + Map values = new HashMap(); + + IntStream.range(0, rsmd.getColumnCount()).forEach(idx -> { + try { + int columnIdx = idx + 1; + // 获取列及对应的列名 + String columnLabel = rsmd.getColumnLabel(columnIdx); + // 根据列名从ResultSet结果集中获得对应的值 + Object columnValue; + + final int columnType = rsmd.getColumnType(columnIdx); + if (SQL_TIME_TYPES.contains(columnType)) { + columnValue = timeHandler(resultSet, columnIdx, columnType); + } else { + columnValue = resultSet.getObject(columnLabel); + } + // 列名为key,列的值为value + values.put(columnLabel, mapper.convertValue(columnValue, String.class)); + + } catch (SQLException ex) { + log.error("putOneResultSetToMap 根据结果集元数据信息转换数据结果集异常 {}", ex.getMessage()); + } + }); + return values; + } + + private String timeHandler(ResultSet resultSet, int columnIdx, int columnType) throws SQLException { + String format = StringUtils.EMPTY; + switch (columnType) { + case Types.DATE: + format = getDateFormat(resultSet, columnIdx); + break; + case Types.TIME: + case Types.TIME_WITH_TIMEZONE: + format = getTimeFormat(resultSet, columnIdx); + break; + case Types.TIMESTAMP: + case Types.TIMESTAMP_WITH_TIMEZONE: + format = getTimestampFormat(resultSet, columnIdx); + break; + default: + } + return format; + } + + private String getDateFormat(@NonNull ResultSet resultSet, int columnIdx) throws SQLException { + String formatTime = StringUtils.EMPTY; + final Date date = resultSet.getDate(columnIdx); + if (Objects.nonNull(date)) { + formatTime = DATE.format(date.toLocalDate()); + } + return formatTime; + } + + private String getTimeFormat(@NonNull ResultSet resultSet, int columnIdx) throws SQLException { + String formatTime = StringUtils.EMPTY; + final Time time = resultSet.getTime(columnIdx); + if (Objects.nonNull(time)) { + formatTime = TIME.format(time.toLocalTime()); + } + return formatTime; + } + + private String getTimestampFormat(@NonNull ResultSet resultSet, int columnIdx) throws SQLException { + String formatTime = StringUtils.EMPTY; + final Timestamp timestamp = resultSet.getTimestamp(columnIdx, Calendar.getInstance(TimeZone.getTimeZone("GMT+8"))); + if (Objects.nonNull(timestamp)) { + formatTime = TIMESTAMP.format(timestamp.toLocalDateTime()); + } + return formatTime; + } + /** + * 结果集对象处理器 将结果集数据转换为JSON字符串 + */ + static class ObjectMapperWapper { + + private static final ObjectMapper MAPPER; + + public static ObjectMapper getObjectMapper() { + return MAPPER; + } + + static { + //创建ObjectMapper对象 + MAPPER = new ObjectMapper(); + + //configure方法 配置一些需要的参数 + // 转换为格式化的json 显示出来的格式美化 + MAPPER.enable(SerializationFeature.INDENT_OUTPUT); + + //序列化的时候序列对象的那些属性 + //JsonInclude.Include.NON_DEFAULT 属性为默认值不序列化 + //JsonInclude.Include.ALWAYS 所有属性 + //JsonInclude.Include.NON_EMPTY 属性为 空（“”） 或者为 NULL 都不序列化 + //JsonInclude.Include.NON_NULL 属性为NULL 不序列化 + MAPPER.setSerializationInclusion(JsonInclude.Include.ALWAYS); + + //反序列化时,遇到未知属性会不会报错 + //true - 遇到没有的属性就报错 false - 没有的属性不会管，不会报错 + MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + //如果是空对象的时候,不抛异常 + MAPPER.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + + //修改序列化后日期格式 + MAPPER.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + MAPPER.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); + //处理不同的时区偏移格式 + MAPPER.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + MAPPER.registerModule(new JavaTimeModule()); + + } + } +} \ No newline at end of file diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/RowDataHashHandler.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/RowDataHashHandler.java new file mode 100644 index 0000000..106fa72 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/RowDataHashHandler.java @@ -0,0 +1,45 @@ +package org.opengauss.datachecker.extract.task; + +import org.opengauss.datachecker.common.entry.extract.RowDataHash; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.opengauss.datachecker.extract.util.HashHandler; +import org.opengauss.datachecker.extract.util.MetaDataUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/17 + * @since ：11 + */ +public class RowDataHashHandler { + + /** + * 根据表元数据信息{@code tableMetadata}中列顺序，对查询出的数据结果进行拼接，并对拼接后的结果行哈希计算 + * + * @param tableMetadata 表元数据信息 + * @param dataRowList 查询数据集合 + * @return 返回抽取数据的哈希计算结果 + */ + public List handlerQueryResult(TableMetadata tableMetadata, List> dataRowList) { + + List recordHashList = new ArrayList<>(); + HashHandler hashHandler = new HashHandler(); + List columns = MetaDataUtil.getTableColumns(tableMetadata); + List primarys = MetaDataUtil.getTablePrimaryColumns(tableMetadata); + dataRowList.forEach(rowColumnsValueMap -> { + long rowHash = hashHandler.xx3Hash(rowColumnsValueMap, columns); + + String primaryValue = hashHandler.value(rowColumnsValueMap, primarys); + long primaryHash = hashHandler.xx3Hash(rowColumnsValueMap, primarys); + RowDataHash hashData = new RowDataHash() + .setPrimaryKey(primaryValue) + .setPrimaryKeyHash(primaryHash) + .setRowHash(rowHash); + recordHashList.add(hashData); + }); + return recordHashList; + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/SelectSqlBulder.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/SelectSqlBulder.java new file mode 100644 index 0000000..dd190f0 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/task/SelectSqlBulder.java @@ -0,0 +1,178 @@ +package org.opengauss.datachecker.extract.task; + +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.springframework.util.Assert; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.opengauss.datachecker.extract.task.SelectSqlBulder.QuerySqlMapper.*; + + +/** + * @author wang chao + * @description 数据抽取SQL构建器 + * @date 2022/5/12 19:17 + * @since 11 + **/ +public class SelectSqlBulder { + private static final long OFF_SET_ZERO = 0L; + /** + * 任务执行起始位置 + */ + private final long start; + /** + * 任务执行偏移量 + */ + private final long offset; + /** + * 查询数据schema + */ + private final String schema; + /** + * 表元数据信息 + */ + private final TableMetadata tableMetadata; + + public SelectSqlBulder(TableMetadata tableMetadata, String schema, long start, long offset) { + this.tableMetadata = tableMetadata; + this.start = start; + this.offset = offset; + this.schema = schema; + } + + public String builder() { + Assert.isTrue(Objects.nonNull(tableMetadata), "表元数据信息异常，构建SQL失败"); + List columnsMetas = tableMetadata.getColumnsMetas(); + if (offset == OFF_SET_ZERO) { + return buildSelectSqlOffsetZero(columnsMetas, tableMetadata.getTableName()); + } else { + return buildSelectSqlOffset(tableMetadata, start, offset); + } + } + + /** + * 根据元数据信息构建查询语句 SELECT * FROM test.test1 + * + * @param columnsMetas 列元数据信息 + * @param tableName 表名 + * @return + */ + private String buildSelectSqlOffsetZero(List columnsMetas, String tableName) { + String columnNames = columnsMetas + .stream() + .map(ColumnsMetaData::getColumnName) + .collect(Collectors.joining(DELIMITER)); + return QUERY_OFF_SET_ZERO.replace(COLUMN, columnNames).replace(SCHEMA, schema).replace(TABLE_NAME, tableName); + } + + /** + * 根据元数据和分片信息构建查询语句 + * SELECT * FROM test.test1 WHERE b_number IN (SELECT t.b_number FROM (SELECT b_number FROM test.test1 LIMIT 0,20) t); + * + * @param tableMetadata 表元数据信息 + * @param start 分片查询起始位置 + * @param offset 分片查询位移 + * @return 返回构建的Select语句 + */ + // + private String buildSelectSqlOffset(TableMetadata tableMetadata, long start, long offset) { + List columnsMetas = tableMetadata.getColumnsMetas(); + List primaryMetas = tableMetadata.getPrimaryMetas(); + + String columnNames; + String primaryKey; + String tableName = tableMetadata.getTableName(); + if (primaryMetas.size() == 1) { + columnNames = columnsMetas + .stream() + .map(ColumnsMetaData::getColumnName) + .collect(Collectors.joining(DELIMITER)); + primaryKey = primaryMetas.stream().map(ColumnsMetaData::getColumnName) + .collect(Collectors.joining()); + return QUERY_OFF_SET.replace(COLUMN, columnNames) + .replace(SCHEMA, schema) + .replace(TABLE_NAME, tableName) + .replace(PRIMARY_KEY, primaryKey) + .replace(START, String.valueOf(start)) + .replace(OFFSET, String.valueOf(offset)); + } else { + columnNames = columnsMetas + .stream() + .map(ColumnsMetaData::getColumnName) + .map(counm -> TABLE_ALAIS.concat(counm)) + .collect(Collectors.joining(DELIMITER)); + primaryKey = primaryMetas.stream().map(ColumnsMetaData::getColumnName) + .collect(Collectors.joining(DELIMITER)); + String joinOn = primaryMetas.stream() + .map(ColumnsMetaData::getColumnName) + .map(coumn -> TABLE_ALAIS.concat(coumn).concat(EQUAL_CONDITION).concat(SUB_TABLE_ALAIS).concat(coumn)) + .collect(Collectors.joining(AND_CONDITION)); + return QUERY_MULTIPLE_PRIMARY_KEY_OFF_SET.replace(COLUMN, columnNames) + .replace(SCHEMA, schema) + .replace(TABLE_NAME, tableName) + .replace(PRIMARY_KEY, primaryKey) + .replace(JOIN_ON, joinOn) + .replace(START, String.valueOf(start)) + .replace(OFFSET, String.valueOf(offset)); + } + } + + /** + * 查询SQL构建模版 + */ + interface QuerySqlMapper { + /** + * 表字段 + */ + String COLUMN = ":columnsList"; + + /** + * 表名称 + */ + String TABLE_NAME = ":tableName"; + + /** + * 表主键 + */ + String PRIMARY_KEY = ":primaryKey"; + String SCHEMA = ":schema"; + /** + * 分片查询起始位置 + */ + String START = ":start"; + /** + * 分片查询偏移量 + */ + String OFFSET = ":offset"; + String JOIN_ON = ":joinOn"; + /** + * 无偏移量场景下，查询SQL语句 + */ + String QUERY_OFF_SET_ZERO = "SELECT :columnsList FROM :schema.:tableName"; + /** + * 单一主键场景下，使用偏移量进行分片查询的SQL语句 + */ + String QUERY_OFF_SET = "SELECT :columnsList FROM :schema.:tableName WHERE :primaryKey IN (SELECT t.:primaryKey FROM (SELECT :primaryKey FROM :schema.:tableName LIMIT :start,:offset) t)"; + String QUERY_MULTIPLE_PRIMARY_KEY_OFF_SET = "SELECT :columnsList FROM :schema.:tableName a RIGHT JOIN (SELECT :primaryKey FROM :schema.:tableName LIMIT :start,:offset) b ON :joinOn"; + /** + * SQL语句字段间隔符号 + */ + String DELIMITER = ","; + /** + * SQL语句 相等条件符号 + */ + String EQUAL_CONDITION = "="; + String AND_CONDITION = " and "; + /** + * 表别名 + */ + String TABLE_ALAIS = "a."; + /** + * 子查询结果别名 + */ + String SUB_TABLE_ALAIS = "b."; + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/HashHandler.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/HashHandler.java new file mode 100644 index 0000000..185b958 --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/HashHandler.java @@ -0,0 +1,52 @@ +package org.opengauss.datachecker.extract.util; + +import org.opengauss.datachecker.common.util.HashUtil; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.opengauss.datachecker.extract.constants.ExtConstants.PRIMARY_DELIMITER; + +/** + * 哈希处理器，对查询结果进行哈希计算。 + * @author ：wangchao + * @date ：Created in 2022/6/15 + * @since ：11 + */ +public class HashHandler { + /** + * 根据columns 集合中字段列表集合，在map中查找字段对应值，并对查找到的值进行拼接。 + * + * @param columnsValueMap 字段对应查询数据 + * @param columns 字段名称列表 + * @return 当前Row对应的哈希计算结果 + */ + public long xx3Hash(Map columnsValueMap, List columns) { + if (CollectionUtils.isEmpty(columns)) { + return 0L; + } + StringBuffer sb = new StringBuffer(); + columns.forEach(colunm -> { + if (columnsValueMap.containsKey(colunm)) { + sb.append(columnsValueMap.get(colunm)); + } + }); + return HashUtil.hashChars(sb.toString()); + } + + public String value(Map columnsValueMap, List columns) { + if (CollectionUtils.isEmpty(columns)) { + return ""; + } + List values = new ArrayList<>(); + columns.forEach(colunm -> { + if (columnsValueMap.containsKey(colunm)) { + values.add(columnsValueMap.get(colunm)); + } + }); + return values.stream().map(String::valueOf).collect(Collectors.joining(PRIMARY_DELIMITER)); + } +} diff --git a/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/MetaDataUtil.java b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/MetaDataUtil.java new file mode 100644 index 0000000..37b1b6a --- /dev/null +++ b/datachecker-extract/src/main/java/org/opengauss/datachecker/extract/util/MetaDataUtil.java @@ -0,0 +1,47 @@ +package org.opengauss.datachecker.extract.util; + +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/15 + * @since ：11 + */ +public class MetaDataUtil { + public static List getTableColumns(TableMetadata tableMetadata) { + if (Objects.isNull(tableMetadata)) { + return emptyList(); + } + List columnsMetas = tableMetadata.getColumnsMetas(); + return getTableColumns(columnsMetas); + } + + public static List getTablePrimaryColumns(TableMetadata tableMetadata) { + if (Objects.isNull(tableMetadata)) { + return emptyList(); + } + List primaryMetas = tableMetadata.getPrimaryMetas(); + return getTableColumns(primaryMetas); + } + + private static ArrayList emptyList() { + return new ArrayList<>(0); + } + + private static List getTableColumns(List columnsMetas) { + if (Objects.isNull(columnsMetas)) { + return emptyList(); + } + return columnsMetas.stream() + .sorted(Comparator.comparing(ColumnsMetaData::getOrdinalPosition)) + .map(ColumnsMetaData::getColumnName) + .collect(Collectors.toList()); + } +} diff --git a/datachecker-extract/src/main/resources/application-sink.yml b/datachecker-extract/src/main/resources/application-sink.yml new file mode 100644 index 0000000..faf7ad2 --- /dev/null +++ b/datachecker-extract/src/main/resources/application-sink.yml @@ -0,0 +1,48 @@ +server: + port: 7001 + +debug: false + +logging: + config: classpath:log4j2-sink.xml + +spring: + application: + name: DATACHECKER-EXTRACT-${spring.extract.endpoint} + + extract: + schema: jack # 宿端数据实例 + databaseType: OG # 宿端数据库类型 OG opengauss + endpoint: SINK # 宿端端点类型 + debezium-enable: false #是否开启增量debezium配置 默认不开启 + + datasource: + druid: + dataSourceOne: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://xxxxx:xxx/xxxx?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC + username: xxxxx + password: xxxxxxxx + type: com.alibaba.druid.pool.DruidDataSource + #Spring Boot 默认是不注入这些属性值的，需要自己绑定 + #druid 数据源专有配置 + initialSize: 20 + minIdle: 5 + maxActive: 50 + maxWait: 60000 + timeBetweenEvictionRunsMillis: 60000 + minEvictableIdleTimeMillis: 300000 + #validationQuery: SELECT 1 FROM DUAL + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + poolPreparedStatements: true + + #配置监控统计拦截的filters，stat:监控统计、log4j：日志记录、wall：防御sql注入 + #如果允许时报错 java.lang.ClassNotFoundException: org.apache.log4j.Priority + #则导入 log4j 依赖即可，Maven 地址：https://mvnrepository.com/artifact/log4j/log4j + filters: stat,wall,log4j + maxPoolPreparedStatementPerConnectionSize: 20 + useGlobalDataSourceStat: true + connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 + diff --git a/datachecker-extract/src/main/resources/application-source.yml b/datachecker-extract/src/main/resources/application-source.yml new file mode 100644 index 0000000..b45d37d --- /dev/null +++ b/datachecker-extract/src/main/resources/application-source.yml @@ -0,0 +1,52 @@ +server: + port: 7002 + +debug: false +logging: + config: classpath:log4j2-source.xml + +spring: + application: + name: DATACHECKER-EXTRACT-${spring.extract.endpoint} + + extract: + schema: test # 源端数据实例 + databaseType: MS # 源端数据库类型 MS mysql + endpoint: SOURCE # 源端端点类型 + debezium-enable: true #是否开启增量debezium配置 默认不开启 + debezium-topic: # debezium监听表增量数据，使用单一topic进行增量数据管理 + debezium-groupId: debezium-extract-group # d debezium增量迁移topic ，groupId消费Group设置 + debezium-topic-partitions: 1 # debezium监听topic 分区数量配置 + debezium-tables: # debezium-tables配置debezium监听的表名称列表; 该配置只在源端服务配置并生效 + debezium-time-period: 1 # debezium增量迁移校验 时间周期 24*60 单位分钟 + debezium-num-period: 1000 #debezium增量迁移校验 统计增量变更记录数量阀值，默认值1000 阀值应大于100 + + datasource: + druid: + dataSourceOne: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://xxxxxx:xxx/xxx?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC&allowPublicKeyRetrieval=true + username: xxxxx + password: xxxxxx + type: com.alibaba.druid.pool.DruidDataSource + #Spring Boot 默认是不注入这些属性值的，需要自己绑定 + #druid 数据源专有配置 + initialSize: 20 + minIdle: 5 + maxActive: 50 + maxWait: 60000 + timeBetweenEvictionRunsMillis: 60000 + minEvictableIdleTimeMillis: 300000 + validationQuery: SELECT 1 FROM DUAL + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + poolPreparedStatements: true + + #配置监控统计拦截的filters，stat:监控统计、log4j：日志记录、wall：防御sql注入 + #如果允许时报错 java.lang.ClassNotFoundException: org.apache.log4j.Priority + #则导入 log4j 依赖即可，Maven 地址：https://mvnrepository.com/artifact/log4j/log4j + filters: stat,wall,log4j + maxPoolPreparedStatementPerConnectionSize: 20 + useGlobalDataSourceStat: true + connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 diff --git a/datachecker-extract/src/main/resources/application.yml b/datachecker-extract/src/main/resources/application.yml new file mode 100644 index 0000000..77e7345 --- /dev/null +++ b/datachecker-extract/src/main/resources/application.yml @@ -0,0 +1,53 @@ +debug: false + +spring: + profiles: + active: sink + check: + server-uri: http://127.0.0.1:7000 # 数据校验服务地址 + + + kafka: + properties: + #这个参数指定producer在发送批量消息前等待的时间，当设置此参数后，即便没有达到批量消息的指定大小(batch-size)，到达时间后生产者也会发送批量消息到broker。默认情况下，生产者的发送消息线程只要空闲了就会发送消息，即便只有一条消息。设置这个参数后，发送线程会等待一定的时间，这样可以批量发送消息增加吞吐量，但同时也会增加延迟。 + linger.ms: 10 #默认值：0毫秒，当消息发送比较频繁时，增加一些延迟可增加吞吐量和性能。 + #这个参数指定producer在一个TCP connection可同时发送多少条消息到broker并且等待broker响应，设置此参数较高的值可以提高吞吐量，但同时也会增加内存消耗。另外，如果设置过高反而会降低吞吐量，因为批量消息效率降低。设置为1，可以保证发送到broker的顺序和调用send方法顺序一致，即便出现失败重试的情况也是如此。 + #注意：当前消息符合at-least-once，自kafka1.0.0以后，为保证消息有序以及exactly once，这个配置可适当调大为5。 + max.in.flight.requests.per.connection: 1 #默认值：5，设置为1即表示producer在connection上发送一条消息，至少要等到这条消息被broker确认收到才继续发送下一条，因此是有序的。 + producer: # producer 生产者 + retries: 0 # 重试次数 + acks: 1 # 应答级别:多少个分区副本备份完成时向生产者发送ack确认(可选0、1、all/-1) + batch-size: 16384 # 批量大小 + buffer-memory: 33554432 # 生产端缓冲区大小 + key-serializer: org.apache.kafka.common.serialization.StringSerializer + # value-serializer: com.itheima.demo.config.MySerializer + value-serializer: org.apache.kafka.common.serialization.StringSerializer + + consumer: # consumer消费者 + group-id: checkgroup # 默认的消费组ID + enable-auto-commit: true # 是否自动提交offset + auto-commit-interval: 100 # 提交offset延时(接收到消息后多久提交offset) + + # earliest:当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，从头开始消费 + # latest:当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，消费新产生的该分区下的数据 + # none:topic各分区都存在已提交的offset时，从offset后开始消费；只要有一个分区不存在已提交的offset，则抛出异常 + auto-offset-reset: earliest + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + # value-deserializer: com.itheima.demo.config.MyDeserializer + value-deserializer: org.apache.kafka.common.serialization.StringDeserializer + max-poll-records: 10000 + + +# springdoc 相关配置 +springdoc: + version: '@springdoc.version@' + swagger-ui: + display-request-duration: true + groups-order: DESC + operationsSorter: method + disable-swagger-default-url: true + use-root-path: true + show-actuator: true + group-configs: + - group: stores + paths-to-match: /extract/** \ No newline at end of file diff --git a/datachecker-extract/src/main/resources/log4j2-sink.xml b/datachecker-extract/src/main/resources/log4j2-sink.xml new file mode 100644 index 0000000..c746d9d --- /dev/null +++ b/datachecker-extract/src/main/resources/log4j2-sink.xml @@ -0,0 +1,141 @@ + + + + + + logs/sink + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/datachecker-extract/src/main/resources/log4j2-source.xml b/datachecker-extract/src/main/resources/log4j2-source.xml new file mode 100644 index 0000000..fcf52fc --- /dev/null +++ b/datachecker-extract/src/main/resources/log4j2-source.xml @@ -0,0 +1,143 @@ + + + + + + logs/source + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/ExtractApplicationTests.java b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/ExtractApplicationTests.java new file mode 100644 index 0000000..f3141a2 --- /dev/null +++ b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/ExtractApplicationTests.java @@ -0,0 +1,13 @@ +package org.opengauss.datachecker.extract; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class ExtractApplicationTests { + + @Test + void contextLoads() { + } + +} diff --git a/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/cache/MetaDataCacheTest.java b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/cache/MetaDataCacheTest.java new file mode 100644 index 0000000..1c07f53 --- /dev/null +++ b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/cache/MetaDataCacheTest.java @@ -0,0 +1,33 @@ +package org.opengauss.datachecker.extract.cache; + +import org.junit.jupiter.api.Test; +import org.opengauss.datachecker.extract.service.MetaDataService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.PostConstruct; + + +@SpringBootTest +public class MetaDataCacheTest { + + @Autowired + private MetaDataService metadataService; + + @PostConstruct + public void init() { + MetaDataCache.initCache(); + MetaDataCache.putMap(metadataService.queryMetaDataOfSchema()); + } + + @Test + public void getTest() { + System.out.println(MetaDataCache.get("client")); + } + + @Test + public void getAllKeysTest() { + System.out.println(MetaDataCache.getAllKeys()); + } + +} diff --git a/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/cache/TestByteXOR.java b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/cache/TestByteXOR.java new file mode 100644 index 0000000..fadc29b --- /dev/null +++ b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/cache/TestByteXOR.java @@ -0,0 +1,61 @@ +package org.opengauss.datachecker.extract.cache; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.IntStream; + +/** + * @author ：wangchao + * @date ：Created in 2022/5/14 + * @since ：11 + */ +public class TestByteXOR { + + @Test + public void testXOR() { + long old = 0L; + for (int i = 0; i < 63; i++) { + old = byteXor(old, i); + System.out.println("0 ," + i + " =" + old + " Long.toBinaryString()" + Long.toBinaryString(old)); + } + } + + @Test + public void testBinaryArray() { + Map map = new HashMap(); + long old = 0L; + byte byteVal = 0; + for (int i = 0; i < 63; i++) { + old = byteXor(old, i); + System.out.println("0 ," + i + " =" + old + " Long.toBinaryString()" + Long.toBinaryString(old)); + } + } + + @Test + public void testIntStream() { + IntStream.range(1, 10).forEach(idx -> { + System.out.println("range " + idx); + }); + IntStream.rangeClosed(1, 10).forEach(idx -> { + System.out.println("rangeClosed " + idx); + }); + + IntStream.rangeClosed(1, 10) + .filter(i -> i == 6) + .count() + ; + } + + /** + * long 64为 该方法计算后63标识符保存数据状态。 + * + * @param value + * @param index + * @return + */ + long byteXor(long value, int index) { + return (value | (1L << index)); + } +} diff --git a/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/config/DataSourceTest.java b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/config/DataSourceTest.java new file mode 100644 index 0000000..ea6d815 --- /dev/null +++ b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/config/DataSourceTest.java @@ -0,0 +1,73 @@ +package org.opengauss.datachecker.extract.config; + +import com.alibaba.druid.pool.DruidDataSource; +import org.junit.jupiter.api.Test; +import org.opengauss.datachecker.extract.ExtractApplication; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.ApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; + +@SpringBootTest (classes = ExtractApplication.class) +public class DataSourceTest { + + @Autowired + private ApplicationContext applicationContext; + + + @Test + public void contextLoadTest() throws SQLException { + DruidDataSource dataSourceOne = (DruidDataSource)applicationContext.getBean("dataSourceOne"); +// DruidDataSource dataSourceTwo = (DruidDataSource)applicationContext.getBean("dataSourceTwo"); +// DruidDataSource dataSourceThree = (DruidDataSource)applicationContext.getBean("dataSourceThree"); + + System.out.println("dataSourceOne " + dataSourceOne.getClass()); + System.out.println("dataSourceOne " + dataSourceOne.getConnection()); + System.out.println("druid dataSourceOne 最大连接数 ：" + dataSourceOne.getMaxActive()); + System.out.println("druid dataSourceOne 最大初始化连接数 ：" + dataSourceOne.getInitialSize()); + + System.out.println(" =========================================== "); +// +// System.out.println("dataSourceTwo " + dataSourceTwo.getClass()); +// System.out.println("dataSourceTwo " + dataSourceTwo.getConnection()); +// System.out.println("druid dataSourceTwo 最大连接数 ：" + dataSourceTwo.getMaxActive()); +// System.out.println("druid dataSourceTwo 最大初始化连接数 ：" + dataSourceTwo.getInitialSize()); +// +// System.out.println(" =========================================== "); +// +// System.out.println("dataSourceThree " + dataSourceThree.getClass()); +// System.out.println("dataSourceThree " + dataSourceThree.getConnection()); +// System.out.println("druid dataSourceThree 最大连接数 ：" + dataSourceThree.getMaxActive()); +// System.out.println("druid dataSourceThree 最大初始化连接数 ：" + dataSourceThree.getInitialSize()); + + dataSourceOne.close(); +// dataSourceTwo.close(); +// dataSourceThree.close(); + + } + + @Test + public void JdbcTemplateTest() { + + JdbcTemplate JdbcTemplateOne = (JdbcTemplate)applicationContext.getBean("JdbcTemplateOne"); +// JdbcTemplate JdbcTemplateTwo = (JdbcTemplate)applicationContext.getBean("JdbcTemplateTwo"); +// JdbcTemplate dataSourceThree = (JdbcTemplate)applicationContext.getBean("JdbcTemplateThree"); +// +// List> listTwo = JdbcTemplateTwo.queryForList("select * from client"); +// for (Map map : listTwo) { +// System.out.println(map); +// } +// +// System.out.println("======================================================================================"); +// List> listThree = dataSourceThree.queryForList("select * from client"); +// for (Map map : listThree) { +// System.out.println(map); +// } + } +} diff --git a/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/dao/DataBaseMetaDataDAOImplTests.java b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/dao/DataBaseMetaDataDAOImplTests.java new file mode 100644 index 0000000..1631597 --- /dev/null +++ b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/dao/DataBaseMetaDataDAOImplTests.java @@ -0,0 +1,37 @@ +package org.opengauss.datachecker.extract.dao; + +import org.junit.jupiter.api.Test; +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.sql.SQLException; +import java.util.List; + +@SpringBootTest +class DataBaseMetaDataDAOImplTests { + + @Autowired + private MetaDataDAO mysqlMetadataDAO; + + @Test + void queryTableMetadata() throws SQLException { + List tableMetadata = mysqlMetadataDAO.queryTableMetadata(); + for (TableMetadata metadata : tableMetadata) { + System.out.println(metadata.toString()); + } + } + + @Test + void queryColumnMetadata() throws SQLException { + List tableMetadata = mysqlMetadataDAO.queryTableMetadata(); + for (TableMetadata metadata : tableMetadata) { + List columnsMetadata = mysqlMetadataDAO.queryColumnMetadata(metadata.getTableName()); + for (ColumnsMetaData colMetadata : columnsMetadata) { + System.out.println(colMetadata.toString()); + } + } + + } +} diff --git a/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/dao/enums/EnumTest.java b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/dao/enums/EnumTest.java new file mode 100644 index 0000000..28abcf5 --- /dev/null +++ b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/dao/enums/EnumTest.java @@ -0,0 +1,27 @@ +package org.opengauss.datachecker.extract.dao.enums; + +import org.junit.jupiter.api.Test; +import org.opengauss.datachecker.common.entry.enums.DataSourceType; +import org.opengauss.datachecker.common.util.EnumUtil; +import org.springframework.boot.test.context.SpringBootTest; + + +@SpringBootTest +public class EnumTest { + + @Test + void testEnum() { + DataSourceType type = DataSourceType.Sink; + + System.out.println(type); + System.out.println(type.equals(DataSourceType.valueOf("Sink"))); + System.out.println(EnumUtil.valueOfIgnoreCase(DataSourceType.class,"Sinkl")); + + System.out.println(EnumUtil.valueOf(DataSourceType.class,"Sinkl")); + System.out.println(EnumUtil.valueOf(DataSourceType.class,"Sink")); + + System.out.println(EnumUtil.valueOf(DataSourceType.class,"sink")); + + System.out.println(EnumUtil.valueOfIgnoreCase(DataSourceType.class,"sink")); + } +} diff --git a/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/service/MetaDataServiceTest.java b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/service/MetaDataServiceTest.java new file mode 100644 index 0000000..0b44a34 --- /dev/null +++ b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/service/MetaDataServiceTest.java @@ -0,0 +1,24 @@ +package org.opengauss.datachecker.extract.service; + +import org.junit.jupiter.api.Test; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.sql.SQLException; +import java.util.Map; + +@SpringBootTest +public class MetaDataServiceTest { + + @Autowired + private MetaDataService metaDataService; + + @Test + void queryMetadataOfSourceDBSchema() throws SQLException { + Map stringTableMetadataMap = metaDataService.queryMetaDataOfSchema(); + for (TableMetadata metadata : stringTableMetadataMap.values()) { + System.out.println(metadata.toString()); + } + } +} diff --git a/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/task/ExtractTaskBuilderTest.java b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/task/ExtractTaskBuilderTest.java new file mode 100644 index 0000000..80575a8 --- /dev/null +++ b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/task/ExtractTaskBuilderTest.java @@ -0,0 +1,36 @@ +package org.opengauss.datachecker.extract.task; + +import org.junit.jupiter.api.Test; +import org.opengauss.datachecker.common.entry.extract.ExtractTask; +import org.opengauss.datachecker.extract.cache.MetaDataCache; +import org.opengauss.datachecker.extract.service.MetaDataService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.PostConstruct; +import java.util.List; +import java.util.Set; + +@SpringBootTest +public class ExtractTaskBuilderTest { + + @Autowired + private ExtractTaskBuilder extractTaskBuilder; + @Autowired + private MetaDataService metadataService; + + @PostConstruct + public void init() { + MetaDataCache.initCache(); + MetaDataCache.putMap(metadataService.queryMetaDataOfSchema()); + } + + @Test + public void builderTest() { + Set tables = MetaDataCache.getAllKeys(); + List extractTasks = extractTaskBuilder.builder(tables); + for (ExtractTask task : extractTasks) { + System.out.println(task); + } + } +} diff --git a/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/task/SelectSqlBulderTest.java b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/task/SelectSqlBulderTest.java new file mode 100644 index 0000000..d4b5893 --- /dev/null +++ b/datachecker-extract/src/test/java/org/opengauss/datachecker/extract/task/SelectSqlBulderTest.java @@ -0,0 +1,112 @@ +package org.opengauss.datachecker.extract.task; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opengauss.datachecker.common.entry.enums.ColumnKey; +import org.opengauss.datachecker.common.entry.extract.ColumnsMetaData; +import org.opengauss.datachecker.common.entry.extract.TableMetadata; + +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SelectSqlBulderTest { + + @Mock + private TableMetadata mockTableMetadata; + + private SelectSqlBulder selectSqlBulderUnderTest; + + @BeforeEach + void setUp() { + selectSqlBulderUnderTest = new SelectSqlBulder(mockTableMetadata, "test",0L, 0L); + } + + @Test + void testBuilder() { + + // Configure TableMetadata.getColumnsMetas(...). + final ColumnsMetaData columnsMeta1 = new ColumnsMetaData(); + columnsMeta1.setTableName("tableName"); + columnsMeta1.setColumnName("columnName1"); + columnsMeta1.setColumnType("columnType"); + columnsMeta1.setDataType("dataType"); + columnsMeta1.setOrdinalPosition(2); + final List columnsMetaData = List.of(columnsMeta1); + when(mockTableMetadata.getColumnsMetas()).thenReturn(columnsMetaData); + when(mockTableMetadata.getTableName()).thenReturn("tableName"); + // Run the test + final String result = selectSqlBulderUnderTest.builder(); + // Verify the results + assertThat(result).isEqualTo("SELECT columnName1 FROM tableName"); + } + + + @Test + void testBuilderOffSet() { + selectSqlBulderUnderTest = new SelectSqlBulder(mockTableMetadata, "test",0L, 1000L); + // Setup + // Configure TableMetadata.getPrimaryMetas(...). + final ColumnsMetaData columnsMetaPri = new ColumnsMetaData(); + columnsMetaPri.setTableName("tableName"); + columnsMetaPri.setColumnName("columnName1"); + columnsMetaPri.setColumnType("columnType"); + columnsMetaPri.setDataType("dataType"); + columnsMetaPri.setOrdinalPosition(1); + columnsMetaPri.setColumnKey(ColumnKey.PRI); + final List primaryList = List.of(columnsMetaPri); + when(mockTableMetadata.getPrimaryMetas()).thenReturn(primaryList); + // Configure TableMetadata.getColumnsMetas(...). + final ColumnsMetaData columnsMeta1 = new ColumnsMetaData(); + columnsMeta1.setTableName("tableName"); + columnsMeta1.setColumnName("columnName2"); + columnsMeta1.setColumnType("columnType"); + columnsMeta1.setDataType("dataType"); + columnsMeta1.setOrdinalPosition(2); + final List columnsMetaData = List.of(columnsMetaPri, columnsMeta1); + when(mockTableMetadata.getColumnsMetas()).thenReturn(columnsMetaData); + when(mockTableMetadata.getTableName()).thenReturn("tableName"); + // Run the test + final String result = selectSqlBulderUnderTest.builder(); + // Verify the results + assertThat(result).isEqualTo("SELECT columnName1,columnName2 FROM tableName WHERE columnName1 IN (SELECT t.columnName1 FROM (SELECT columnName1 FROM tableName LIMIT 0,1000) t)"); + } + + @Test + void testBuilderMuliPrimaryLeyOffSet() { + selectSqlBulderUnderTest = new SelectSqlBulder(mockTableMetadata, "test",0L, 1000L); + // Setup + // Configure TableMetadata.getPrimaryMetas(...). + final ColumnsMetaData columnsMetaPri = new ColumnsMetaData(); + columnsMetaPri.setTableName("tableName"); + columnsMetaPri.setColumnName("columnName1"); + columnsMetaPri.setColumnType("columnType"); + columnsMetaPri.setDataType("dataType"); + columnsMetaPri.setOrdinalPosition(1); + columnsMetaPri.setColumnKey(ColumnKey.PRI); + + // Configure TableMetadata.getColumnsMetas(...). + final ColumnsMetaData columnsMeta1 = new ColumnsMetaData(); + columnsMeta1.setTableName("tableName"); + columnsMeta1.setColumnName("columnName2"); + columnsMeta1.setColumnType("columnType"); + columnsMeta1.setDataType("dataType"); + columnsMeta1.setOrdinalPosition(2); + columnsMeta1.setColumnKey(ColumnKey.PRI); + final List columnsMetaData = List.of(columnsMetaPri, columnsMeta1); + final List primaryList = List.of(columnsMetaPri, columnsMeta1); + when(mockTableMetadata.getPrimaryMetas()).thenReturn(primaryList); + when(mockTableMetadata.getColumnsMetas()).thenReturn(columnsMetaData); + when(mockTableMetadata.getTableName()).thenReturn("tableName"); + // Run the test + final String result = selectSqlBulderUnderTest.builder(); + // Verify the results + assertThat(result).isEqualTo("SELECT a.columnName1,a.columnName2 FROM tableName a RIGHT JOIN (SELECT columnName1,columnName2 FROM tableName LIMIT 0,1000) b ON a.columnName1=b.columnName1 and a.columnName2=b.columnName2"); + } +} diff --git a/datachecker-mock-data/.gitignore b/datachecker-mock-data/.gitignore new file mode 100644 index 0000000..549e00a --- /dev/null +++ b/datachecker-mock-data/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/datachecker-mock-data/pom.xml b/datachecker-mock-data/pom.xml new file mode 100644 index 0000000..3971392 --- /dev/null +++ b/datachecker-mock-data/pom.xml @@ -0,0 +1,80 @@ + + + 4.0.0 + + org.opengauss + openGauss-tools-datachecker-performance + 0.0.1 + ../pom.xml + + datachecker-mock-data + 0.0.1 + datachecker-mock-data + datachecker-mock-data + + + 11 + 0.0.1 + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.opengauss + datachecker-common + ${datachecker.version} + + + org.springframework.boot + spring-boot-starter-log4j2 + + + org.springframework.boot + spring-boot-starter-data-jdbc + + + mysql + mysql-connector-java + provided + + + com.alibaba + druid + + + org.springframework.boot + spring-boot-starter-validation + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + mysql + mysql-connector-java + + + + + + + + diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/MockDataApplication.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/MockDataApplication.java new file mode 100644 index 0000000..e952af5 --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/MockDataApplication.java @@ -0,0 +1,13 @@ +package org.opengauss.datachecker.extract; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class MockDataApplication { + + public static void main(String[] args) { + SpringApplication.run(MockDataApplication.class, args); + } + +} diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/TableRowCount.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/TableRowCount.java new file mode 100644 index 0000000..dc4f57b --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/TableRowCount.java @@ -0,0 +1,17 @@ +package org.opengauss.datachecker.extract; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.experimental.Accessors; + +/** + * @author ：wangchao + * @date ：Created in 2022/6/6 + * @since ：11 + */ +@Data +@AllArgsConstructor +public class TableRowCount { + private String tableName; + private long count; +} diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/AsyncConfig.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/AsyncConfig.java new file mode 100644 index 0000000..7bc84d6 --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/AsyncConfig.java @@ -0,0 +1,37 @@ +package org.opengauss.datachecker.extract.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * @author wang chao + * @date 2022/5/8 19:17 + * @since 11 + **/ +@Configuration +@EnableAsync +public class AsyncConfig { + + @Bean("threadPoolTaskExecutor") + public ThreadPoolTaskExecutor doAsyncExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + // 核心线程数， 当前机器的核心数 线程池创建时初始化线程数量 + executor.setCorePoolSize(16); + // 最大线程数：线程池最大的线程数，只有在缓冲队列满了之后才会申请超过核心线程数的线程 + executor.setMaxPoolSize(32); + // 缓冲队列： 用来缓冲执行任务的队列 + executor.setQueueCapacity(4000); + //允许线程空闲时间 + executor.setKeepAliveSeconds(60); + // 线程池名称前缀 + executor.setThreadNamePrefix("extract-"); + // 缓冲队列满了之后的拒绝策略： + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + executor.initialize(); + return executor; + } +} diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/DruidDataSourceConfig.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/DruidDataSourceConfig.java new file mode 100644 index 0000000..41e0d6e --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/DruidDataSourceConfig.java @@ -0,0 +1,46 @@ +package org.opengauss.datachecker.extract.config; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.support.http.StatViewServlet; +import com.alibaba.druid.support.http.WebStatFilter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.boot.web.servlet.ServletRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +@Configuration +public class DruidDataSourceConfig { + + /** + * 

+     *  将自定义的 Druid数据源添加到容器中，不再让 Spring Boot 自动创建
+     *  绑定全局配置文件中的 druid 数据源属性到 com.alibaba.druid.pool.DruidDataSource从而让它们生效
+     *  @ConfigurationProperties(prefix = "spring.datasource")：作用就是将 全局配置文件中
+     *  前缀为 spring.datasource的属性值注入到 com.alibaba.druid.pool.DruidDataSource 的同名参数中
+     *
+ * + * @return + */ + @Primary + @Bean("dataSourceOne") + @ConfigurationProperties(prefix = "spring.datasource.druid") + public DataSource druidDataSourceOne() { + return new DruidDataSource(); + } + + + @Bean("jdbcTemplateOne") + public JdbcTemplate jdbcTemplateOne(@Qualifier("dataSourceOne") DataSource dataSourceOne) { + return new JdbcTemplate(dataSourceOne); + } + +} diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/ExtractConfig.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/ExtractConfig.java new file mode 100644 index 0000000..e5e0869 --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/ExtractConfig.java @@ -0,0 +1,25 @@ +package org.opengauss.datachecker.extract.config; + +import lombok.Getter; +import org.opengauss.datachecker.common.entry.enums.DataBaseType; +import org.opengauss.datachecker.common.entry.enums.DataSourceType; +import org.springframework.context.annotation.Configuration; + +@Configuration +@Getter +public class ExtractConfig { + /** + * schema的初始化值由配置文件加载 + */ + private final String schema = "test"; + + /** + * schema的初始化值由配置文件加载 + */ + private final DataSourceType dataSourceType = DataSourceType.Source; + + /** + * schema的初始化值由配置文件加载 + */ + private final DataBaseType databaseType = DataBaseType.MS; +} diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/SpringDocConfig.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/SpringDocConfig.java new file mode 100644 index 0000000..91a1e8b --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/config/SpringDocConfig.java @@ -0,0 +1,60 @@ +package org.opengauss.datachecker.extract.config; + +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.parameters.HeaderParameter; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.springdoc.core.customizers.OpenApiCustomiser; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.util.ReflectionUtils; +import org.springframework.web.servlet.config.annotation.InterceptorRegistration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import java.lang.reflect.Field; +import java.util.List; + +/** + * swagger2配置 + * http://localhost:8080/swagger-ui/index.html + * + * @author ：wangchao + * @date ：Created in 2022/5/17 + * @since ：11 + */ + +/** + * 2021/8/13 + */ + +@Configuration +public class SpringDocConfig implements WebMvcConfigurer { + @Bean + public OpenAPI mallTinyOpenAPI() { + return new OpenAPI() + .info(new Info() + .title("数据打桩服务") + .description("数据打桩服务 自动化执行测试表创建以及数据插入 API") + .version("v1.0.0")); + } + + /** + * 通用拦截器排除设置，所有拦截器都会自动加springdoc-opapi相关的资源排除信息，不用在应用程序自身拦截器定义的地方去添加，算是良心解耦实现。 + */ + @SuppressWarnings("unchecked") + @Override + public void addInterceptors(InterceptorRegistry registry) { + try { + Field registrationsField = FieldUtils.getField(InterceptorRegistry.class, "registrations", true); + List registrations = (List) ReflectionUtils.getField(registrationsField, registry); + if (registrations != null) { + for (InterceptorRegistration interceptorRegistration : registrations) { + interceptorRegistration.excludePathPatterns("/springdoc**/**"); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/controller/ExtractMockController.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/controller/ExtractMockController.java new file mode 100644 index 0000000..422261a --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/controller/ExtractMockController.java @@ -0,0 +1,66 @@ +package org.opengauss.datachecker.extract.controller; + +import io.swagger.v3.oas.annotations.Parameter; +import org.hibernate.validator.constraints.Range; +import org.opengauss.datachecker.extract.TableRowCount; +import org.opengauss.datachecker.extract.service.ExtractMockDataService; +import org.opengauss.datachecker.extract.service.ExtractMockTableService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + + +@RestController +public class ExtractMockController { + @Autowired + private ExtractMockDataService extractMockDataService; + @Autowired + private ExtractMockTableService extractMockTableService; + + /** + * 根据名称创建数据库表 表字段目前为固定字段 + * + * @param tableName 创建表名 + * @return + * @throws Exception + */ + @PostMapping("/mock/createTable") + public String createTable(@RequestParam("tableName") String tableName) throws Exception { + return extractMockTableService.createTable(tableName); + } + + @GetMapping("/mock/query/all/table/count") + public List getAllTableCount() { + return extractMockTableService.getAllTableCount(); + } + + /** + * 向指定表名称，采用多线程方式批量插入指定数据量的Mock数据 + * + * @param tableName 表名 + * @param totalCount 插入数据总量 + * @param threadCount 线程数 最大线程总数不能超过2000 ，超过2000可能会导致数据丢失 + * @return + */ + @PostMapping("/batch/mock/data") + public String batchMockData(@Parameter(name = "tableName", description = "待插入数据表名") @RequestParam("tableName") String tableName, + @Parameter(name = "totalCount", description = "待插入数据总量") @RequestParam("totalCount") long totalCount, + @Parameter(name = "threadCount", description = "多线程插入，设置线程总数") + @Range(min = 1, max = 30, message = "设置的线程总数必须在[1-30]之间") + @RequestParam("threadCount") int threadCount, + @Parameter(name = "createTable", description = "是否创建表") @RequestParam("createTable") boolean createTable) { + try { + if (createTable) { + extractMockTableService.createTable(tableName); + } + extractMockDataService.batchMockData(tableName, totalCount, threadCount); + } catch (Exception throwables) { + System.err.println(throwables.getMessage()); + } + return "OK"; + } +} diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/ExtractMockDataService.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/ExtractMockDataService.java new file mode 100644 index 0000000..a0995b9 --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/ExtractMockDataService.java @@ -0,0 +1,53 @@ +package org.opengauss.datachecker.extract.service; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.extract.service.thread.ExtractMockDataThread; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; + +import javax.sql.DataSource; +import java.util.concurrent.Future; + +@Slf4j +@Service +public class ExtractMockDataService { + + /** + * 限制最大线程总数 + */ + private static final int MAX_THREAD_COUNT = 100; + + @Autowired + private ThreadPoolTaskExecutor threadPoolTaskExecutor; + + @Autowired + private DataSource dataSourceOne; + + /** + * 向指定表名称，采用多线程方式批量插入指定数据量的Mock数据 + * + * @param tableName 待插入数据的表名称 + * @param totalCount 插入记录总数 + * @param threadCount 插入记录线程总数 + */ + public void batchMockData(String tableName, long totalCount, int threadCount) { + try { + Assert.isTrue(threadCount < MAX_THREAD_COUNT, "设置的线程总数不能超过最大线程总数"); + long batchCount = totalCount / threadCount; + + log.info("plan batch insert thread, tableName = {}, threadCount = {} ,totalCount = {} , batchCount = {}", tableName, threadCount, totalCount, batchCount); + for (int i = 0; i < threadCount; i++) { + if (i == (threadCount - 1)) { + batchCount = batchCount + totalCount % threadCount; + } + threadPoolTaskExecutor.submit(new ExtractMockDataThread(dataSourceOne, tableName, batchCount, i + 1)); + + } + } catch (Exception throwables) { + log.error("=============", throwables.getMessage()); + } + + } +} diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/ExtractMockTableService.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/ExtractMockTableService.java new file mode 100644 index 0000000..8768ac9 --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/ExtractMockTableService.java @@ -0,0 +1,103 @@ +package org.opengauss.datachecker.extract.service; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.util.ThreadUtil; +import org.opengauss.datachecker.extract.TableRowCount; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author wang chao + * @date 2022/5/8 19:27 + * @since 11 + **/ +@Service +@Slf4j +public class ExtractMockTableService { + @Autowired + protected JdbcTemplate jdbcTemplateOne; + + @Autowired + private ThreadPoolTaskExecutor threadPoolTaskExecutor; + + /** + * 自动创建指定表 + * + * @param tableName 创建表 + * @return + * @throws Exception 目前对于表名重复未做处理，表名重复这里直接抛出异常信息 + */ + public String createTable(String tableName) throws Exception { + jdbcTemplateOne.execute(MockMapper.CREATE.replace(":TABLENAME", tableName)); + return tableName; + } + + private static final String SQL_QUERY_TABLE = "SELECT table_name from information_schema.TABLES WHERE table_schema='test' "; + + public List getAllTableCount() { + long start = System.currentTimeMillis(); + List tableRowCountList = new ArrayList<>(); + final List tableNameList = jdbcTemplateOne.queryForList(SQL_QUERY_TABLE, String.class); + if (CollectionUtils.isEmpty(tableNameList)) { + return new ArrayList<>(); + } + String sqlQueryTableRowCount = "select count(1) rowCount from test.%s"; + tableNameList.stream().forEach(tableName -> { + threadPoolTaskExecutor.submit(() -> { + final Long rowCount = jdbcTemplateOne.queryForObject(String.format(sqlQueryTableRowCount, tableName), Long.class); + tableRowCountList.add(new TableRowCount(tableName, rowCount)); + }); + }); + + while (tableRowCountList.size() != tableNameList.size()) { + ThreadUtil.sleep(10); + } + + final long sum = tableRowCountList.stream().mapToLong(TableRowCount::getCount).sum(); + tableRowCountList.add(new TableRowCount("all_table_total", sum)); + tableRowCountList.sort((o1, o2) -> (int) (o1.getCount() - o2.getCount())); + + long end = System.currentTimeMillis(); + + System.out.println(" query cost time =" + (end - start) + " sec"); + return tableRowCountList; + } + + /** + * 构建创建表SQL语句 + */ + interface MockMapper { + String CREATE = "CREATE TABLE :TABLENAME (\n" + + "\t b_number VARCHAR(30) NOT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_type VARCHAR(20) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_user VARCHAR(20) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_int INT(10) NULL DEFAULT NULL,\n" + + "\t b_bigint BIGINT(19) NULL DEFAULT '0',\n" + + "\t b_text TEXT NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_longtext LONGTEXT NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_date DATE NULL DEFAULT NULL,\n" + + "\t b_datetime DATETIME NULL DEFAULT NULL,\n" + + "\t b_timestamp TIMESTAMP NULL DEFAULT NULL,\n" + + "\t b_attr1 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_attr2 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_attr3 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_attr4 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_attr5 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_attr6 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_attr7 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_attr8 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_attr9 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\t b_attr10 VARCHAR(255) NULL DEFAULT NULL COLLATE 'utf8mb4_0900_ai_ci',\n" + + "\tPRIMARY KEY (`b_number`) USING BTREE\n" + + ")\n" + + " COLLATE='utf8mb4_0900_ai_ci'\n" + + " ENGINE=InnoDB ;\n"; + + } +} diff --git a/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/thread/ExtractMockDataThread.java b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/thread/ExtractMockDataThread.java new file mode 100644 index 0000000..cf965b9 --- /dev/null +++ b/datachecker-mock-data/src/main/java/org/opengauss/datachecker/extract/service/thread/ExtractMockDataThread.java @@ -0,0 +1,134 @@ +package org.opengauss.datachecker.extract.service.thread; + +import lombok.extern.slf4j.Slf4j; +import org.opengauss.datachecker.common.util.IdWorker; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +/** + * @author wang chao + * @description 数据抽取服务，基础数据打桩 + * @date 2022/5/8 19:27 + * @since 11 + **/ +@Slf4j +public class ExtractMockDataThread implements Runnable { + protected static final int MAX_INSERT_ROW_COUNT = 10000; + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + protected JdbcTemplate jdbcTemplate; + protected String tableName; + protected long maxRowCount; + protected int taskSn; + + + /** + * 线程构造函数 + * + * @param dataSource 数据源注入 + * @param tableName 表名注入 + * @param maxRowCount 当前线程最大插入记录总数 + * @param taskSn 线程任务序列号 + */ + public ExtractMockDataThread(DataSource dataSource, String tableName, long maxRowCount, int taskSn) { + jdbcTemplate = new JdbcTemplate(dataSource); + this.tableName = tableName; + this.maxRowCount = maxRowCount; + this.taskSn = taskSn; + } + + @Override + public void run() { + batchMockData(tableName, maxRowCount, taskSn); + } + + + public void batchMockData(String tableName, long threadMaxRowCount, int taskSn) { + try { + log.info("batch start insert: table:{},counut={},taskSn={}", tableName, threadMaxRowCount, taskSn); + long batchInsertCount = threadMaxRowCount; + long insertedCount = 0; + // 循环分批次插入数据，限制SQL每次插入记录的最大数据量 + while (batchInsertCount >= MAX_INSERT_ROW_COUNT) { + String batchSql = buildBatchSql(tableName, MAX_INSERT_ROW_COUNT, taskSn); + jdbcTemplate.batchUpdate(batchSql); + batchInsertCount = batchInsertCount - MAX_INSERT_ROW_COUNT; + insertedCount += MAX_INSERT_ROW_COUNT; + log.info("batch insert:threadMaxRowCount={},taskSn={},insertedCount={}", threadMaxRowCount, taskSn, insertedCount); + } + // 循环分批次插入数据，最后一个批次的数据插入 + if (batchInsertCount > 0 && batchInsertCount < MAX_INSERT_ROW_COUNT) { + String batchSql = buildBatchSql(tableName, batchInsertCount, taskSn); + jdbcTemplate.batchUpdate(batchSql); + insertedCount += batchInsertCount; + log.info("batch insert:totalRowCount={},taskSn={},insertedCount={}", threadMaxRowCount, taskSn, insertedCount); + } + log.info("batch end insert: table:{},counut={},taskSn={}", tableName, threadMaxRowCount, taskSn); + } catch (Exception e) { + log.error("batch insert:{},{},{}", threadMaxRowCount, taskSn, e.getMessage()); + } + } + + private String buildBatchSql(String tableName, long rowCount, int ordler) { + StringBuffer sb = new StringBuffer(MockMapper.INSERT.replace(":TABLENAME", tableName)); + for (int i = 0; i < rowCount; i++) { + String id = IdWorker.nextId(String.valueOf(ordler)); + sb.append("(") + // b_number + .append("'").append(id).append("',") + // b_type + .append("'type_01',") + // b_user + .append("'user_02',") + //b_int + .append("1,") + //b_bigint + .append("32,") + // b_text + .append("'b_text_").append(id).append("',") + // b_longtext + .append("'b_longtext_").append(id).append("',") + // b_date + .append("'").append(DATE_FORMATTER.format(LocalDate.now())).append("',") + // b_datetime + .append("'").append(DATE_TIME_FORMATTER.format(LocalDateTime.now())).append("',") + // b_timestamp + .append("'").append(DATE_TIME_FORMATTER.format(LocalDateTime.now())).append("',") + // b_attr1 + .append("'b_attr1_").append(id).append("',") + // b_attr2 + .append("'b_attr2_").append(id).append("',") + // b_attr3 + .append("'b_attr3_").append(id).append("',") + // b_attr4 + .append("'b_attr4_").append(id).append("',") + // b_attr5 + .append("'b_attr5_").append(id).append("',") + // b_attr6 + .append("'b_attr6_").append(id).append("',") + // b_attr7 + .append("'b_attr7_").append(id).append("',") + // b_attr8 + .append("'b_attr8_").append(id).append("',") + // b_attr9 + .append("'b_attr9_").append(id).append("',") + // b_attr10 + .append("'b_attr10_").append(id).append("'") + .append(")") + .append(","); + } + int length = sb.length(); + sb.deleteCharAt(length - 1); + return sb.toString(); + } + + + interface MockMapper { + String INSERT = "INSERT INTO :TABLENAME VALUES "; + } +} diff --git a/datachecker-mock-data/src/main/resources/application.yml b/datachecker-mock-data/src/main/resources/application.yml new file mode 100644 index 0000000..88a083e --- /dev/null +++ b/datachecker-mock-data/src/main/resources/application.yml @@ -0,0 +1,40 @@ +server: + port: 8081 + +logging: + config: classpath:log4j2.xml + +spring: + application: + name: datachecker-extract + + datasource: + druid: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://192.168.0.114:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC&allowPublicKeyRetrieval=true + username: root + password: Huawei@123 + type: com.alibaba.druid.pool.DruidDataSource + #Spring Boot 默认是不注入这些属性值的，需要自己绑定 + #druid 数据源专有配置 + initialSize: 30 + minIdle: 10 + maxActive: 100 + maxWait: 60000 + timeBetweenEvictionRunsMillis: 60000 + minEvictableIdleTimeMillis: 300000 + validationQuery: SELECT 1 FROM DUAL + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + poolPreparedStatements: true + mysql: + usePingMethod: false + + #配置监控统计拦截的filters，stat:监控统计、log4j：日志记录、wall：防御sql注入 + #如果允许时报错 java.lang.ClassNotFoundException: org.apache.log4j.Priority + #则导入 log4j 依赖即可，Maven 地址：https://mvnrepository.com/artifact/log4j/log4j +# filters: stat,wall,log4j +# maxPoolPreparedStatementPerConnectionSize: 20 +# useGlobalDataSourceStat: true +# connectionProperties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 \ No newline at end of file diff --git a/datachecker-mock-data/src/main/resources/log4j2.xml b/datachecker-mock-data/src/main/resources/log4j2.xml new file mode 100644 index 0000000..6df8594 --- /dev/null +++ b/datachecker-mock-data/src/main/resources/log4j2.xml @@ -0,0 +1,89 @@ + + + + + + logs/mock + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/openGauss-tools-datachecker-performance.iml b/openGauss-tools-datachecker-performance.iml new file mode 100644 index 0000000..ad5a890 --- /dev/null +++ b/openGauss-tools-datachecker-performance.iml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..5abb9a1 --- /dev/null +++ b/pom.xml @@ -0,0 +1,110 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 2.6.7 + + + org.opengauss + openGauss-tools-datachecker-performance + 0.0.1 + pom + + openGauss-tools-datachecker-performance + openGauss-tools-datachecker-performance + + + UTF-8 + UTF-8 + 2.6.7 + 2021.0.0 + 11 + 1.2.11 + 1.18.24 + 1.2.75 + 19.0 + 1.6.8 + 0.15 + 8.0.29 + + + datachecker-common + datachecker-extract + datachecker-check + datachecker-mock-data + + + + + + + org.springframework.cloud + spring-cloud-dependencies + ${spring.cloud.version} + pom + import + + + + org.springdoc + springdoc-openapi-ui + ${springdoc.openapi.ui.version} + + + + mysql + mysql-connector-java + ${mysql.connector.java.version} + provided + + + com.alibaba + druid + ${druid.version} + + + + org.projectlombok + lombok + true + ${lombok.version} + + + org.apache.commons + commons-collections4 + 4.4 + + + + com.google.guava + guava + ${guava.version} + + + + net.openhft + zero-allocation-hashing + ${zero.allocation.hashing.version} + + + + com.alibaba + fastjson + ${fastjson.version} + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + -- Gitee