From c855565ed8198276c13c8e4f2fc0c38812bcad96 Mon Sep 17 00:00:00 2001 From: zwtmichael Date: Fri, 31 Dec 2021 16:59:50 +0800 Subject: [PATCH 01/38] database status notify callback Signed-off-by: zwtmichael --- .../common/include/runtime_context.h | 5 ++ .../common/src/runtime_context_impl.cpp | 19 ++++++ .../common/src/runtime_context_impl.h | 8 +++ .../libs/distributeddb/include/types_export.h | 3 + .../include/kv_store_delegate_manager.h | 2 + .../src/kv_store_delegate_manager.cpp | 5 ++ .../syncer/src/generic_syncer.cpp | 2 +- .../syncer/src/single_ver_kv_syncer.cpp | 4 ++ .../syncer/src/single_ver_syncer.cpp | 4 ++ ...distributeddb_single_ver_p2p_sync_test.cpp | 66 +++++++++++++++++++ 10 files changed, 117 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h index ac76d7a6d..5d704f94b 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h @@ -105,6 +105,11 @@ public: // Notify TIME_CHANGE_EVENT. virtual void NotifyTimeStampChanged(TimeOffset offset) const = 0; + + virtual void SetStoreStatusNotifier(const StoreStatusNotifier ¬ifier) = 0; + + virtual void NotifyDatabaseStatusChange(const std::string &userId, const std::string &appId, const std::string &storeId, + const std::string &deviceId, bool onlineStatus) = 0; protected: RuntimeContext() = default; virtual ~RuntimeContext() {} diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp index faaa671e6..f3d08cc60 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp @@ -575,4 +575,23 @@ bool RuntimeContextImpl::IsCommunicatorAggregatorValid() const } return true; } + +void RuntimeContextImpl::SetStoreStatusNotifier(const StoreStatusNotifier ¬ifier) +{ + std::unique_lock writeLock(databaseStatusCallbackMutex_); + databaseStatusNotifyCallback_ = notifier; + LOGI("SetStoreStatusNotifier ok"); +} + +void RuntimeContextImpl::NotifyDatabaseStatusChange(const std::string &userId, const std::string &appId, + const std::string &storeId, const std::string &deviceId, bool onlineStatus) +{ + ScheduleTask([this, userId, appId, storeId, deviceId, onlineStatus] { + std::shared_lock autoLock(databaseStatusCallbackMutex_); + if (databaseStatusNotifyCallback_) { + LOGI("start notify database status:%d", onlineStatus); + databaseStatusNotifyCallback_(userId, appId, storeId, deviceId, onlineStatus); + } + }); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h index 4d78e9261..72ff30636 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h @@ -97,6 +97,11 @@ public: // Notify TIME_CHANGE_EVENT. void NotifyTimeStampChanged(TimeOffset offset) const override; + void SetStoreStatusNotifier(const StoreStatusNotifier ¬ifier) override; + + void NotifyDatabaseStatusChange(const std::string &userId, const std::string &appId, const std::string &storeId, + const std::string &deviceId, bool onlineStatus) override; + private: static constexpr int MAX_TP_THREADS = 10; // max threads of the task pool. static constexpr int MIN_TP_THREADS = 1; // min threads of the task pool. @@ -142,6 +147,9 @@ private: std::shared_ptr systemApiAdapter_; mutable std::mutex lockStatusLock_; // Mutex for lockStatusObserver_. LockStatusObserver *lockStatusObserver_; + + mutable std::shared_mutex databaseStatusCallbackMutex_{}; + StoreStatusNotifier databaseStatusNotifyCallback_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/include/types_export.h b/services/distributeddataservice/libs/distributeddb/include/types_export.h index 6e474e734..97687358a 100755 --- a/services/distributeddataservice/libs/distributeddb/include/types_export.h +++ b/services/distributeddataservice/libs/distributeddb/include/types_export.h @@ -119,6 +119,9 @@ using PermissionCheckCallback = std::function; +using StoreStatusNotifier = std::function; // status, 1: online, 0: offline + enum AutoLaunchStatus { WRITE_OPENED = 1, WRITE_CLOSED = 2, diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_delegate_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_delegate_manager.h index e1613e077..47c6a847c 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_delegate_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_delegate_manager.h @@ -102,6 +102,8 @@ public: DB_API static DBStatus SetProcessSystemAPIAdapter(const std::shared_ptr &adapter); + DB_API static void SetStoreStatusNotifier(const StoreStatusNotifier ¬ifier); + private: // Check if the dataDir is safe arg. diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp index a0c0cde7c..f144d07ab 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp @@ -578,4 +578,9 @@ DBStatus KvStoreDelegateManager::SetProcessSystemAPIAdapter(const std::shared_pt { return TransferDBErrno(RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(adapter)); } + +void KvStoreDelegateManager::SetStoreStatusNotifier(const StoreStatusNotifier ¬ifier) +{ + RuntimeContext::GetInstance()->SetStoreStatusNotifier(notifier); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp index dd6f1094c..a6da32f77 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp @@ -189,7 +189,7 @@ int GenericSyncer::Sync(const SyncParma ¶m) int GenericSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) { auto *operation = - new (std::nothrow) SyncOperation(syncId, param.devices, param.mode, param.onComplete, param.wait); + new (std::nothrow) SyncOperation(syncId, param.devices, param.mode, param.onComplete, param.wait); if (operation == nullptr) { SubQueuedSyncSize(); return -E_OUT_OF_MEMORY; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp index d1c8732ac..f3efa44d7 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp @@ -125,6 +125,10 @@ void SingleVerKVSyncer::LocalDataChanged(int notifyEvent) void SingleVerKVSyncer::RemoteDataChanged(const std::string &device) { LOGI("[SingleVerKVSyncer] device online dev %s", STR_MASK(device)); + std::string userId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); + std::string appId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); + std::string storeId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); + RuntimeContext::GetInstance()->NotifyDatabaseStatusChange(userId, appId, storeId, device, true); // while remote db is online again, need to do abilitySync static_cast(syncEngine_)->SetIsNeedResetAbilitySync(device, true); if (autoSyncEnable_) { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp index cbe12a6d8..af13a3a95 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp @@ -21,6 +21,10 @@ namespace DistributedDB { void SingleVerSyncer::RemoteDeviceOffline(const std::string &device) { LOGI("[SingleVerRelationalSyncer] device offline dev %s", STR_MASK(device)); + std::string userId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); + std::string appId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); + std::string storeId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); + RuntimeContext::GetInstance()->NotifyDatabaseStatusChange(userId, appId, storeId, device, false); RefObject::IncObjRef(syncEngine_); static_cast(syncEngine_)->OfflineHandleByDevice(device); RefObject::DecObjRef(syncEngine_); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp index 5ae7a97af..882a6b947 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp @@ -2222,3 +2222,69 @@ HWTEST_F(DistributedDBSingleVerP2PSyncTest, SametimeSync001, TestSize.Level3) g_kvDelegatePtr->Get(key, actualValue); EXPECT_EQ(actualValue, value); } + +/** + * @tc.name: DatabaseOnlineCallback001 + * @tc.desc: check database status notify online callback + * @tc.type: FUNC + * @tc.require: AR000CQS3S SR000CQE0B + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSyncTest, DatabaseOnlineCallback001, TestSize.Level1) +{ + /** + * @tc.steps: step1. SetStoreStatusNotifier + * @tc.expected: step1. SetStoreStatusNotifier ok + */ + std::string targetDev = "DEVICE_X"; + bool isCheckOk = false; + auto databaseStatusNotifyCallback = [targetDev, &isCheckOk] (std::string userId, + std::string appId, std::string storeId, const std::string deviceId, bool onlineStatus) -> void { + if (userId == USER_ID && appId == APP_ID && storeId == STORE_ID && deviceId == targetDev + && onlineStatus == true) { + isCheckOk = true; + }}; + g_mgr.SetStoreStatusNotifier(databaseStatusNotifyCallback); + /** + * @tc.steps: step2. trigger device online + * @tc.expected: step2. check callback ok + */ + g_communicatorAggregator->OnlineDevice(targetDev); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_TIME / 20)); + EXPECT_EQ(isCheckOk, true); + StoreStatusNotifier nullCallback; + g_mgr.SetStoreStatusNotifier(nullCallback); +} + +/** + * @tc.name: DatabaseOfflineCallback001 + * @tc.desc: check database status notify online callback + * @tc.type: FUNC + * @tc.require: AR000CQS3S SR000CQE0B + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSyncTest, DatabaseOfflineCallback001, TestSize.Level1) +{ + /** + * @tc.steps: step1. SetStoreStatusNotifier + * @tc.expected: step1. SetStoreStatusNotifier ok + */ + std::string targetDev = "DEVICE_X"; + bool isCheckOk = false; + auto databaseStatusNotifyCallback = [targetDev, &isCheckOk] (std::string userId, + std::string appId, std::string storeId, const std::string deviceId, bool onlineStatus) -> void { + if (userId == USER_ID && appId == APP_ID && storeId == STORE_ID && deviceId == targetDev + && onlineStatus == false) { + isCheckOk = true; + }}; + g_mgr.SetStoreStatusNotifier(databaseStatusNotifyCallback); + /** + * @tc.steps: step2. trigger device offline + * @tc.expected: step2. check callback ok + */ + g_communicatorAggregator->OfflineDevice(targetDev); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_TIME / 20)); + EXPECT_EQ(isCheckOk, true); + StoreStatusNotifier nullCallback; + g_mgr.SetStoreStatusNotifier(nullCallback); +} -- Gitee From 6f0783223fb763f856e6faf737131a7d0a4dc308 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 5 Jan 2022 16:43:41 +0800 Subject: [PATCH 02/38] Add remove device data Signed-off-by: lianhuix --- .../distributeddb/common/include/db_common.h | 2 + .../relational/relational_schema_object.h | 6 + .../distributeddb/common/src/db_common.cpp | 5 + .../distributeddb/common/src/json_object.cpp | 2 +- .../common/src/query_expression.cpp | 2 +- .../relational/relational_schema_object.cpp | 38 ++- .../distributeddb/include/query_expression.h | 2 +- .../relational/relational_store_delegate.h | 8 +- .../relational/relational_store_manager.h | 7 +- .../relational_store_delegate_impl.cpp | 26 +- .../relational_store_delegate_impl.h | 5 +- .../relational/relational_store_manager.cpp | 63 +--- .../relational/relational_sync_able_storage.h | 8 + .../include/relational_db_sync_interface.h | 4 + .../include/relational_store_connection.h | 3 + .../src/relational_sync_able_storage.cpp | 20 ++ .../storage/src/sqlite/query_object.cpp | 2 +- .../relational/sqlite_relational_store.cpp | 79 ++++- .../relational/sqlite_relational_store.h | 5 +- .../sqlite_relational_store_connection.cpp | 20 ++ .../sqlite_relational_store_connection.h | 3 + ...single_ver_relational_storage_executor.cpp | 108 +++++- ...e_single_ver_relational_storage_executor.h | 6 + .../distributeddb/syncer/src/ability_sync.cpp | 3 +- .../libs/distributeddb/test/BUILD.gn | 14 + ...ibuteddb_relational_schema_object_test.cpp | 109 ++++++ .../common/distributeddb_tools_unit_test.cpp | 27 ++ .../common/distributeddb_tools_unit_test.h | 6 + ...stributeddb_interfaces_relational_test.cpp | 311 ++++++++++++++++++ ...distributeddb_relational_get_data_test.cpp | 1 - ...ributeddb_relational_ver_p2p_sync_test.cpp | 4 - ...rtual_relational_ver_sync_db_interface.cpp | 11 + ...virtual_relational_ver_sync_db_interface.h | 4 + 33 files changed, 814 insertions(+), 100 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_common.h b/services/distributeddataservice/libs/distributeddb/common/include/db_common.h index 3dedbdd0d..8d2d793da 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_common.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_common.h @@ -54,6 +54,8 @@ public: const std::string &storeId); static std::string StringMasking(const std::string &oriStr, size_t remain = 3); // remain 3 unmask + + static std::string GetDistributedTableName(const std::string &device, const std::string &tableName); }; // Define short macro substitute for original long expression for convenience of using diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index d13b4c165..75a5e8af1 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -136,8 +136,12 @@ public: void AddRelationalTable(const TableInfo& tb); + void RemoveRelationalTable(const std::string &tableName); + const std::map &GetTables() const; + std::vector GettableNames() const; + TableInfo GetTable(const std::string& tableName) const; private: @@ -159,6 +163,8 @@ private: int ParseCheckTableIndex(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTablePrimaryKey(const JsonObject &inJsonObject, TableInfo &resultTable); + void GenerateSchemaString(); + bool isValid_ = false; // set to true after parse success from string or add at least one relational table SchemaType schemaType_ = SchemaType::NONE; // Default NONE std::string schemaString_; // The minified and valid schemaString diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp index c04b39406..3ffe156b8 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp @@ -311,4 +311,9 @@ std::string DBCommon::StringMasking(const std::string &oriStr, size_t remain) } return oriStr; } + +std::string DBCommon::GetDistributedTableName(const std::string &device, const std::string &tableName) +{ + return DBConstant::RELATIONAL_PREFIX + tableName + "_" + DBCommon::TransferHashString(device); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index 7e9b2b470..595dde63a 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -403,7 +403,7 @@ int JsonObject::GetArrayContentOfStringOrStringArray(const FieldPath &inPath, int errCode = E_OK; const Json::Value &valueNode = GetJsonValueByFieldPath(inPath, errCode); if (errCode != E_OK) { - LOGE("[Json][GetArrayContent] Get JsonValue Fail=%d.", errCode); + LOGW("[Json][GetArrayContent] Get JsonValue Fail=%d.", errCode); return errCode; } if (valueNode.type() != Json::ValueType::arrayValue) { diff --git a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp index 20e6988d6..90ff60ec3 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp @@ -212,7 +212,7 @@ const std::string &QueryExpression::GetTableName() return tableName_; } -bool QueryExpression::IsTableNameSpacified() const +bool QueryExpression::IsTableNameSpecified() const { return isTableNameSpecified_; } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index 285221bd4..08fc8ac7d 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -94,7 +94,6 @@ void FieldInfo::SetColumnId(int cid) cid_ = cid; } -// return field define string like ("fieldName": "MY INT(21), NOT NULL, DEFAULT 123") std::string FieldInfo::ToAttributeString() const { std::string attrStr = "\"" + fieldName_ + "\": {"; @@ -303,6 +302,7 @@ JsonObject TableInfo::ToJsonObject() const std::string TableInfo::ToTableInfoString() const { std::string attrStr; + attrStr += "{"; attrStr += R"("NAME": ")" + tableName_ + "\","; AddFieldDefineString(attrStr); attrStr += R"("AUTOINCREMENT": )"; @@ -316,6 +316,7 @@ std::string TableInfo::ToTableInfoString() const attrStr += R"("PRIMARY_KEY": ")" + primaryKey_ + "\","; } AddIndexDefineString(attrStr); + attrStr += "}"; return attrStr; } @@ -410,9 +411,31 @@ int RelationalSchemaObject::ParseFromSchemaString(const std::string &inSchemaStr return E_OK; } +void RelationalSchemaObject::GenerateSchemaString() +{ + schemaString_ = {}; + schemaString_ += "{"; + schemaString_ += R"("SCHEMA_VERSION":"2.0",)"; + schemaString_ += R"("SCHEMA_TYPE":"RELATIVE",)"; + schemaString_ += R"("TABLES":[)"; + for (const auto &it : tables_) { + schemaString_ += it.second.ToTableInfoString() + ","; + } + schemaString_ += R"(])"; + schemaString_ += "}"; +} + void RelationalSchemaObject::AddRelationalTable(const TableInfo &tb) { tables_[tb.GetTableName()] = tb; + schemaString_ = {}; +} + + +void RelationalSchemaObject::RemoveRelationalTable(const std::string &tableName) +{ + tables_.erase(tableName); + schemaString_ = {}; } const std::map &RelationalSchemaObject::GetTables() const @@ -420,6 +443,15 @@ const std::map &RelationalSchemaObject::GetTables() cons return tables_; } +std::vector RelationalSchemaObject::GettableNames() const +{ + std::vector tableNames; + for (const auto &it : tables_) { + tableNames.emplace_back(it.first); + } + return tableNames; +} + TableInfo RelationalSchemaObject::GetTable(const std::string &tableName) const { auto it = tables_.find(tableName); @@ -661,7 +693,7 @@ int RelationalSchemaObject::ParseCheckTableUnique(const JsonObject &inJsonObject { std::vector uniqueArray; int errCode = inJsonObject.GetArrayContentOfStringOrStringArray(FieldPath {"UNIQUE"}, uniqueArray); - if (errCode != E_OK) { + if (errCode != E_OK && errCode != -E_INVALID_PATH) { // UNIQUE is not necessary LOGE("[RelationalSchema][Parse] Get unique array failed: %d.", errCode); return -E_SCHEMA_PARSE_FAIL; } @@ -683,7 +715,7 @@ int RelationalSchemaObject::ParseCheckTableIndex(const JsonObject &inJsonObject, { std::map tableFields; int errCode = inJsonObject.GetSubFieldPathAndType(FieldPath {"INDEX"}, tableFields); - if (errCode != E_OK) { + if (errCode != E_OK && errCode != -E_INVALID_PATH) { // INDEX is not necessary LOGE("[RelationalSchema][Parse] Get schema TABLES INDEX failed: %d.", errCode); return -E_SCHEMA_PARSE_FAIL; } diff --git a/services/distributeddataservice/libs/distributeddb/include/query_expression.h b/services/distributeddataservice/libs/distributeddb/include/query_expression.h index 48084148c..241500207 100755 --- a/services/distributeddataservice/libs/distributeddb/include/query_expression.h +++ b/services/distributeddataservice/libs/distributeddb/include/query_expression.h @@ -114,7 +114,7 @@ public: void SetTableName(const std::string &tableName); const std::string &GetTableName(); - bool IsTableNameSpacified() const; + bool IsTableNameSpecified() const; std::string GetSuggestIndex() const; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index fbf038564..c839e6fc0 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -30,10 +30,6 @@ public: // split mode }; - DB_API virtual DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) = 0; - - DB_API virtual DBStatus RemoveDeviceData(const std::string &device) = 0; - DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName) = 0; DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, @@ -42,7 +38,9 @@ public: DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, const Query &query, SyncStatusCallback &onComplete, bool wait) = 0; - DB_API virtual DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) = 0; + DB_API virtual DBStatus RemoveDeviceData(const std::string &device) = 0; + + DB_API virtual DBStatus RemoveDeviceData(const std::string &device, const std::string &tableName) = 0; }; } // namespace DistributedDB #endif // RELATIONAL_STORE_DELEGATE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 6a06e4878..53c5d64c4 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -26,6 +26,9 @@ namespace DistributedDB { class RelationalStoreManager final { public: + // Only calculate the table name with device hash, no guarantee for the table exists + DB_API static std::string GetDistributedTableName(const std::string &device, const std::string &tableName); + DB_API RelationalStoreManager(const std::string &appId, const std::string &userId); DB_API ~RelationalStoreManager() = default; @@ -37,9 +40,7 @@ public: DB_API DBStatus OpenStore(const std::string &path, const std::string &storeId, const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate); - DB_API DBStatus CloseStore(RelationalStoreDelegate *store); - - DB_API DBStatus DeleteStore(const std::string &path); + DB_API DBStatus CloseStore(RelationalStoreDelegate *&store); DB_API static void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index a4d279cdf..c2b36a1fc 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -37,11 +37,6 @@ RelationalStoreDelegateImpl::~RelationalStoreDelegateImpl() conn_ = nullptr; }; -DBStatus RelationalStoreDelegateImpl::Pragma(PragmaCmd cmd, PragmaData ¶mData) -{ - return NOT_SUPPORT; -} - DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) { @@ -50,7 +45,7 @@ DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devic DBStatus RelationalStoreDelegateImpl::RemoveDeviceData(const std::string &device) { - return NOT_SUPPORT; + return RemoveDeviceData(device, {}); } DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName) @@ -91,9 +86,24 @@ DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devic return OK; } -DBStatus RelationalStoreDelegateImpl::RemoveDevicesData(const std::string &tableName, const std::string &device) +DBStatus RelationalStoreDelegateImpl::RemoveDeviceData(const std::string &device, const std::string &tableName) { - return NOT_SUPPORT; + if (conn_ == nullptr) { + LOGE("Invalid connection for operation!"); + return DB_ERROR; + } + + if (device.empty()) { + LOGE("[RelationalStore Delegate] Remove device data with unspecified device name."); + return INVALID_ARGS; + } + + int errCode = conn_->RemoveDeviceData(device, tableName); + if (errCode != E_OK) { + LOGW("[RelationalStore Delegate] remove device data failed:%d", errCode); + return TransferDBErrno(errCode); + } + return OK; } DBStatus RelationalStoreDelegateImpl::Close() diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 76eb8defe..701953eaa 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -29,8 +29,6 @@ public: DISABLE_COPY_ASSIGN_MOVE(RelationalStoreDelegateImpl); - DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) override; - DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) override; @@ -41,10 +39,11 @@ public: DBStatus CreateDistributedTable(const std::string &tableName) override; - DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) override; + DBStatus RemoveDeviceData(const std::string &device, const std::string &tableName) override; // For connection DBStatus Close(); + void SetReleaseFlag(bool flag); private: diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index a72f5667a..5fcf4916d 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -96,7 +96,7 @@ DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const return OK; } -DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *store) +DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *&store) { if (store == nullptr) { return INVALID_ARGS; @@ -114,65 +114,12 @@ DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *store) return OK; } -static int RemoveFile(const std::string &fileName) +std::string RelationalStoreManager::GetDistributedTableName(const std::string &device, const std::string &tableName) { - if (!OS::CheckPathExistence(fileName)) { - return E_OK; + if (device.empty() || tableName.empty()) { + return {}; } - - if (OS::RemoveFile(fileName.c_str()) != E_OK) { - LOGE("Remove file failed:%d", errno); - return -E_REMOVE_FILE; - } - return E_OK; -} - -static int RemoveDB(const std::string &path) -{ - if (RemoveFile(path) != E_OK) { - LOGE("Remove the db file failed:%d", errno); - return -E_REMOVE_FILE; - } - - std::string dbFile = path + "-wal"; - if (RemoveFile(dbFile) != E_OK) { - LOGE("Remove the wal file failed:%d", errno); - return -E_REMOVE_FILE; - } - - dbFile = path + "-shm"; - if (RemoveFile(dbFile) != E_OK) { - LOGE("Remove the shm file failed:%d", errno); - return -E_REMOVE_FILE; - } - return E_OK; -} - -DBStatus RelationalStoreManager::DeleteStore(const std::string &path) -{ - if (path.empty()) { - LOGE("Invalid store info for deleting"); - return INVALID_ARGS; - } - - std::string identifier = userId_ + "-" + appId_ + "-" + path; - std::string hashIdentifier = DBCommon::TransferHashString(identifier); - - auto *manager = RelationalStoreInstance::GetInstance(); - int errCode = manager->CheckDatabaseFileStatus(hashIdentifier); - if (errCode != E_OK) { - LOGE("The store is busy!"); - return BUSY; - } - - // RemoveDB - errCode = RemoveDB(path); - if (errCode == E_OK) { - LOGI("Database deleted successfully!"); - return OK; - } - LOGE("Delete the kv store error:%d", errCode); - return TransferDBErrno(errCode); + return DBCommon::GetDistributedTableName(device, tableName); } void RelationalStoreManager::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index e9e6562b4..2d16239f2 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -103,6 +103,12 @@ public: return E_OK; } + int CreateDistributedDeviceTable(const std::string &device, const RelationalSyncStrategy &syncStrategy) override; + + int RegisterSchemaChangedCallback(const std::function &callback) override; + + void NotifySchemaChanged(); + private: SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode, OperatePerm perm) const; void ReleaseHandle(SQLiteSingleVerRelationalStorageExecutor *&handle) const; @@ -121,6 +127,8 @@ private: TimeStamp currentMaxTimeStamp_ = 0; KvDBProperties properties; mutable std::mutex maxTimeStampMutex_; + + std::function onSchemaChanged_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h index 94749dba3..b9dde8e43 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h @@ -38,6 +38,10 @@ public: virtual int LocalDataChanged(int notifyEvent, std::vector &queryObj) = 0; virtual int SchemaChanged(int notifyEvent) = 0; + + virtual int CreateDistributedDeviceTable(const std::string &device, const RelationalSyncStrategy &syncStrategy) = 0; + + virtual int RegisterSchemaChangedCallback(const std::function &callback) = 0; }; } #endif // RELATIONAL_STORE diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index f3dd78fc6..0259d2d36 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -53,6 +53,9 @@ public: virtual int CreateDistributedTable(const std::string &tableName) = 0; virtual int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) = 0; + virtual int RemoveDeviceData(const std::string &device) = 0; + virtual int RemoveDeviceData(const std::string &device, const std::string &tableName) = 0; + protected: // Get the stashed 'KvDB_ pointer' without ref. template diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index 70418aa13..883477569 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -459,5 +459,25 @@ int RelationalSyncAbleStorage::SchemaChanged(int notifyEvent) { return -E_NOT_SUPPORT; } + +int RelationalSyncAbleStorage::CreateDistributedDeviceTable(const std::string &device, + const RelationalSyncStrategy &syncStrategy) +{ + return E_OK; +} + +int RelationalSyncAbleStorage::RegisterSchemaChangedCallback(const std::function &callback) +{ + onSchemaChanged_ = callback; + return E_OK; +} + +void RelationalSyncAbleStorage::NotifySchemaChanged() +{ + if (onSchemaChanged_) { + LOGD("Notify relational schema was changed"); + onSchemaChanged_(); + } +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp index 8c465a1c8..3fec4df8d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp @@ -74,7 +74,7 @@ QueryObject::QueryObject(const Query &query) prefixKey_ = queryExpressions.GetPreFixKey(); suggestIndex_ = queryExpressions.GetSuggestIndex(); tableName_ = queryExpressions.GetTableName(); - isTableNameSpecified_ = queryExpressions.IsTableNameSpacified(); + isTableNameSpecified_ = queryExpressions.IsTableNameSpecified(); } QueryObject::QueryObject(const std::list &queryObjNodes, const std::vector &prefixKey) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index fea33f026..7bda3da7a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -142,12 +142,47 @@ int SQLiteRelationalStore::SaveSchemaToMeta() int SQLiteRelationalStore::SaveLogTableVersionToMeta() { LOGD("save log table version to meta table, key: %s, val: %s", LOG_TABLE_VERSION_KEY, LOG_TABLE_VERSION_1); + const Key logVersionKey(LOG_TABLE_VERSION_KEY, LOG_TABLE_VERSION_KEY + strlen(LOG_TABLE_VERSION_KEY)); + Value logVersionVal(LOG_TABLE_VERSION_1, LOG_TABLE_VERSION_1 + strlen(LOG_TABLE_VERSION_1)); + int errCode = storageEngine_->PutMetaData(logVersionKey, logVersionVal); + if (errCode != E_OK) { + LOGE("save log table version to meta table failed. %d", errCode); + } return E_OK; } int SQLiteRelationalStore::CleanDistributedDeviceTable() { - return E_OK; + int errCode = E_OK; + auto handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + + std::lock_guard lock(schemaMutex_); + RelationalSchemaObject schema = properties_.GetSchema(); + std::vector missingTables; + errCode = handle->CkeckAndCleanDistributedTable(schema.GettableNames(), missingTables); + if (errCode == E_OK) { + errCode = handle->Commit(); + // Remove non-existent tables from the schema + for (const auto &tableName : missingTables) { + schema.RemoveRelationalTable(tableName); + } + properties_.SetSchema(schema); + } else { + LOGE("Check distributed table failed. %d", errCode); + (void)handle->Rollback(); + } + + ReleaseHandle(handle); + return errCode; } int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) @@ -338,12 +373,52 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) ReleaseHandle(handle); return errCode; } + (void)handle->Commit(); schema.AddRelationalTable(table); properties_.SetSchema(schema); - (void)handle->Commit(); + storageEngine_->NotifySchemaChanged(); ReleaseHandle(handle); return SaveSchemaToMeta(); } + +int SQLiteRelationalStore::RemoveDeviceData(const std::string &device, const std::string &tableName) +{ + { + std::lock_guard lock(schemaMutex_); + std::map tables = properties_.GetSchema().GetTables(); + if (!tableName.empty() && tables.find(tableName) == tables.end()) { + LOGW("Remove device data with table name which is not a distributed table or not exist."); + return E_OK; + } + } + + int errCode = E_OK; + auto *handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + + errCode = handle->DeleteDistributedDeviceTable(device, tableName); + if (errCode != E_OK) { + LOGE("delete device data failed. %d", errCode); + (void)handle->Rollback(); + } else { + (void)handle->Commit(); + errCode = syncEngine_->EraseDeviceWaterMark(device, true); + if (errCode != E_OK) { + LOGE("Erase device water mark failed. %d", errCode); + } + } + + ReleaseHandle(handle); + return errCode; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index ebb3fb2b5..06ea31501 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -52,6 +52,9 @@ public: } int CreateDistributedTable(const std::string &tableName); + + int RemoveDeviceData(const std::string &device, const std::string &tableName); + private: void ReleaseResources(); @@ -77,12 +80,12 @@ private: std::atomic connectionCount_ = 0; std::vector> closeNotifiers_; + mutable std::mutex schemaMutex_; RelationalDBProperties properties_; mutable std::mutex initalMutex_; bool isInitialized_ = false; - mutable std::mutex schemaMutex_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index de529540b..7b4b66ed8 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -150,6 +150,26 @@ int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &t return errCode; } +int SQLiteRelationalStoreConnection::RemoveDeviceData(const std::string &device) +{ + return RemoveDeviceData(device, {}); +} + +int SQLiteRelationalStoreConnection::RemoveDeviceData(const std::string &device, const std::string &tableName) +{ + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[RelationalConnection] store is null, get DB failed!"); + return -E_INVALID_CONNECTION; + } + + int errCode = store->RemoveDeviceData(device, tableName); + if (errCode != E_OK) { + LOGE("[RelationalConnection] remove device data failed. %d", errCode); + } + return errCode; +} + int SQLiteRelationalStoreConnection::Pragma(int cmd, void *parameter) // reserve for interface function fix { return E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 552447e76..c5b86bf61 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -40,6 +40,9 @@ public: int CreateDistributedTable(const std::string &tableName) override; int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) override; + int RemoveDeviceData(const std::string &device) override; + int RemoveDeviceData(const std::string &device, const std::string &tableName) override; + protected: int Pragma(int cmd, void *parameter) override; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index fd7657987..1042e3adf 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -427,11 +427,8 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const { - // naturalbase_rdb_aux_userTableName_deviceHash - // tableName std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = DBConstant::RELATIONAL_PREFIX + object.GetTableName() + "_" + - DBCommon::TransferStringToHex(devName); + const std::string tableName = DBCommon::GetDistributedTableName(devName, object.GetTableName()); TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode == -E_NOT_FOUND) { @@ -526,8 +523,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem) { std::string devName = DBCommon::TransferHashString(dataItem.dev); - const std::string tableName = DBConstant::RELATIONAL_PREFIX + table_.GetTableName() + "_" + - DBCommon::TransferStringToHex(devName); + const std::string tableName = DBCommon::GetDistributedTableName(devName, table_.GetTableName()); std::string hashKey = std::string(dataItem.hashKey.begin(), dataItem.hashKey.end()); std::string sql = "DELETE FROM " + tableName + " WHERE calc_hash(" + table_.GetPrimaryKey() + ")=" + hashKey + ";"; sqlite3_stmt *stmt = nullptr; @@ -727,5 +723,105 @@ int SQLiteSingleVerRelationalStorageExecutor::CheckDBModeForRelational() } return E_OK; } + +int SQLiteSingleVerRelationalStorageExecutor::DeleteDistributedDeviceTable(const std::string &device, + const std::string &tableName) +{ + if (device.empty() && tableName.empty()) { // device and table name should not both be empty + return -E_INVALID_ARGS; + } + std::string decicePattern = device.empty() ? "%" : DBCommon::TransferHashString(device); + std::string tablePattern = tableName.empty() ? "%" : tableName; + std::string deviceTableName = DBConstant::RELATIONAL_PREFIX + tablePattern + "_" + decicePattern; + + static const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE " + + deviceTableName + ";"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, checkSql, stmt); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; + } + + do { + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + break; + } else if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + LOGE("Get table name failed. %d", errCode); + break; + } + std::string realTableName; + SQLiteUtils::GetColumnTextValue(stmt, 1, realTableName); // 1: table name result column index + std::string deleteSql = "DROP TABLE IF EXISTS " + realTableName + ";"; // drop the found table + int errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, deleteSql); + if (errCode != E_OK) { + LOGE("Delete device data failed. %d", errCode); + break; + } + } while(true); + + SQLiteUtils::ResetStatement(stmt, true, errCode); + return CheckCorruptedStatus(errCode); +} + +int SQLiteSingleVerRelationalStorageExecutor::DeleteDistributedLogTable(const std::string &tableName) +{ + if (tableName.empty()) { + return -E_INVALID_ARGS; + } + std::string logTableName = DBConstant::RELATIONAL_PREFIX + tableName + "_log"; + std::string deleteSql = "DROP TABLE IF EXISTS " + logTableName + ";"; + int errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, deleteSql); + if (errCode != E_OK) { + LOGE("Delete distributed log table failed. %d", errCode); + } + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::CkeckAndCleanDistributedTable(const std::vector &tableNames, + std::vector &missingTables) +{ + if (tableNames.empty()) { + return E_OK; + } + static const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name='?';"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, checkSql, stmt); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; + } + for (const auto &tableName : tableNames) { + errCode = SQLiteUtils::BindTextToStatement(stmt, 1, tableName); // 1: tablename bind index + if (errCode != E_OK) { + LOGE("Bind table name to check distributed table statement failed. %d", errCode); + break; + } + + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { // The table in schema was dropped + errCode = DeleteDistributedDeviceTable({}, tableName); // Clean the auxiliary tables for the dropped table + if (errCode != E_OK) { + LOGE("Delete device tables for missing distributed table failed. %d", errCode); + break; + } + errCode = DeleteDistributedLogTable(tableName); + if (errCode != E_OK) { + LOGE("Delete log tables for missing distributed table failed. %d", errCode); + break; + } + missingTables.emplace_back(tableName); + } else if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + LOGE("Check distributed table failed. %s", errCode); + break; + } + errCode = E_OK; // Check result ok for distributed table is still exists + SQLiteUtils::ResetStatement(stmt, false, errCode); + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + return CheckCorruptedStatus(errCode); +} } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h index ba0057ae1..0f241591f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -60,6 +60,12 @@ public: int CheckDBModeForRelational(); + int DeleteDistributedDeviceTable(const std::string &device, const std::string &tableName); + int DeleteDistributedLogTable(const std::string &tableName); + + int CkeckAndCleanDistributedTable(const std::vector &tableNames, + std::vector &missingTables); + private: int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData) const; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp index f7a591fe3..5efe7b6d7 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp @@ -981,7 +981,7 @@ int AbilitySync::SetAbilityRequestBodyInfo(AbilitySyncRequestPacket &packet, uin SecurityOption option; GetPacketSecOption(option); std::string schemaStr; - uint32_t schemaType; + uint32_t schemaType = 0; if (IsSingleKvVer()) { SchemaObject schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); schemaStr = schemaObj.ToSchemaString(); @@ -989,7 +989,6 @@ int AbilitySync::SetAbilityRequestBodyInfo(AbilitySyncRequestPacket &packet, uin } #ifdef RELATIONAL_STORE if (IsSingleRelationalVer()) { - // todo demo auto schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); schemaStr = schemaObj.ToSchemaString(); schemaType = 0; diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 0a46a901d..ab5fb8e10 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -621,6 +621,18 @@ distributeddb_unittest("DistributedDBSingleVerP2PSubscribeSyncTest") { sources = [ "unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp" ] } +distributeddb_unittest("DistributedInterfacesRelationalTest") { + sources = [ + "unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp", + ] +} + +distributeddb_unittest("DistributedDBRelationalSchemaObjectTest") { + sources = [ + "unittest/common/common/distributeddb_relational_schema_object_test.cpp", + ] +} + ############################################################################### group("unittest") { testonly = true @@ -665,6 +677,7 @@ group("unittest") { ":DistributedDBMultiVerVacuumTest", ":DistributedDBNotificationChainTest", ":DistributedDBParcelTest", + ":DistributedDBRelationalSchemaObjectTest", ":DistributedDBSchemaObjectTest", ":DistributedDBSchemalTest", ":DistributedDBSingleVerMsgScheduleTest", @@ -688,6 +701,7 @@ group("unittest") { ":DistributedDBStorageTransactionRecordTest", ":DistributedDBSyncerDeviceManagerTest", ":DistributedDBTimeSyncTest", + ":DistributedInterfacesRelationalTest", ":RuntimeContextProcessSystemApiAdapterImplTest", ] } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp new file mode 100644 index 000000000..77f194d4e --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OMIT_JSON +#include +#include + +#include "db_errno.h" +#include "distributeddb_tools_unit_test.h" +#include "log_print.h" +#include "relational_schema_object.h" +#include "schema_utils.h" + +using namespace std; +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; + +namespace { + constexpr const char *NORMAL_SCHEMA = R""({ + "SCHEMA_VERSION": "2.0", + "SCHEMA_TYPE": "RELATIVE", + "TABLES": [{ + "NAME": "FIRST", + "DEFINE": { + "field_name1": { + "TYPE": "STRING", + "NOT_NULL": true, + "DEFINE": "abcd" + }, + "field_name2": { + "TYPE": "MYINT(21)", + "NOT_NULL": false, + "DEFINE": "222" + }, + "field_name3": { + "TYPE": "INTGER", + "NOT_NULL": false, + "DEFINE": "1" + } + }, + "AUTOINCREMENT": true, + "UNIQUE": ["field_name1", ["field_name2", "field_name3"]], + "PRIMARY_KEY": "field_name1", + "INDEX": { + "index_name1": ["field_name1", "field_name2"], + "index_name2": ["field_name3"] + } + }, { + "NAME": "SECOND", + "DEFINE": { + "key": { + "TYPE": "BLOB", + "NOT_NULL": true + }, + "value": { + "TYPE": "BLOB", + "NOT_NULL": false + } + }, + "PRIMARY_KEY": "field_name1" + }] + })""; +} + +class DistributedDBRelationalSchemaObjectTest : public testing::Test { +public: + static void SetUpTestCase(void) {}; + static void TearDownTestCase(void) {}; + void SetUp() override ; + void TearDown() override {}; +}; + +void DistributedDBRelationalSchemaObjectTest::SetUp() +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); +} + +/** + * @tc.name: RelationalSchemaTest001 + * @tc.desc: Test relational schema parse from json string + * @tc.type: FUNC + * @tc.require: AR000GK58I + * @tc.author: lianhuix + */ +HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaTest001, TestSize.Level1) +{ + const std::string schemaStr = NORMAL_SCHEMA; + RelationalSchemaObject schemaObj; + int errCode = schemaObj.ParseFromSchemaString(schemaStr); + EXPECT_EQ(errCode, E_OK); + + RelationalSchemaObject schemaObj2; + schemaObj2.ParseFromSchemaString(schemaObj.ToSchemaString()); + EXPECT_EQ(errCode, E_OK); +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp index e15f218f8..88c9ef376 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp @@ -842,4 +842,31 @@ int DistributedDBToolsUnitTest::BuildMessage(const DataSyncMessageInfo &messageI message->SetExternalObject(packet); return E_OK; } + +sqlite3 *RelationalTestUtils::CreateDataBase(const std::string &dbUri) +{ + sqlite3 *db = nullptr; + if (int r = sqlite3_open_v2(dbUri.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr) != SQLITE_OK) { + LOGE("Open database [%s] failed. %d", dbUri.c_str(), r); + if (db != nullptr) { + (void)sqlite3_close_v2(db); + db = nullptr; + } + } + return db; +} + +int RelationalTestUtils::ExecSql(sqlite3 *db, const std::string &sql) +{ + if (db == nullptr || sql.empty()) { + return -E_INVALID_ARGS; + } + char *errMsg = nullptr; + int errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + LOGE("Execute sql failed. %d err: %s", errCode, errMsg); + } + sqlite3_free(errMsg); + return errCode; +} } // namespace DistributedDBUnitTest diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h index d6a8e2903..66484bd62 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h @@ -270,6 +270,12 @@ public: private: std::vector databaseInfoVect_; }; + +class RelationalTestUtils { +public: + static sqlite3 *CreateDataBase(const std::string &dbUri); + static int ExecSql(sqlite3 *db, const std::string &sql); +}; } // namespace DistributedDBUnitTest #endif // DISTRIBUTEDDB_TOOLS_UNIT_TEST_H diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp new file mode 100644 index 000000000..d350dc1cb --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "log_print.h" +#include "relational_store_manager.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { + constexpr const char* DB_SUFFIX = ".db"; + constexpr const char* STORE_ID = "Relational_Store_ID"; + std::string g_testDir; + std::string g_dbDir; + DistributedDB::RelationalStoreManager g_mgr(APP_ID, USER_ID); + + const std::string NORMAL_CREATE_TABLE_SQL = "CREATE TABLE IF NOT EXISTS sync_data(" \ + "key BLOB NOT NULL," \ + "value BLOB," \ + "timestamp INT NOT NULL," \ + "flag INT NOT NULL," \ + "device BLOB," \ + "ori_device BLOB," \ + "hash_key BLOB PRIMARY KEY NOT NULL," \ + "w_timestamp INT);" \ + "CREATE INDEX key_index ON sync_data (key, flag);"; +} + +class DistributedInterfacesRelationalTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedInterfacesRelationalTest::SetUpTestCase(void) +{ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + LOGD("Test dir is %s", g_testDir.c_str()); + g_dbDir = g_testDir + "/"; +} + +void DistributedInterfacesRelationalTest::TearDownTestCase(void) +{ +} + +void DistributedInterfacesRelationalTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); +} + +void DistributedInterfacesRelationalTest::TearDown(void) +{ + DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir); +} + +/** + * @tc.name: RelationalStoreTest001 + * @tc.desc: Test open store and create distributed db + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare db file + * @tc.expected: step1. Return OK. + */ + sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, NORMAL_CREATE_TABLE_SQL), SQLITE_OK); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); + + /** + * @tc.steps:step2. open relational store, create distributed table, close store + * @tc.expected: step2. Return OK. + */ + RelationalStoreDelegate *delegate = nullptr; + DBStatus status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate); + EXPECT_EQ(status, OK); + ASSERT_NE(delegate, nullptr); + + status = delegate->CreateDistributedTable("sync_data"); + EXPECT_EQ(status, OK); + + // test create same table again + status = delegate->CreateDistributedTable("sync_data"); + EXPECT_EQ(status, OK); + + status = g_mgr.CloseStore(delegate); + EXPECT_EQ(status, OK); + ASSERT_EQ(delegate, nullptr); + + /** + * @tc.steps:step3. drop sync_data table + * @tc.expected: step3. Return OK. + */ + db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "drop table sync_data;"), SQLITE_OK); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); + + /** + * @tc.steps:step4. open again, check auxiliary should be delete + * @tc.expected: step4. Return OK. + */ + delegate = nullptr; + status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate); + EXPECT_EQ(status, OK); + ASSERT_NE(delegate, nullptr); + status = g_mgr.CloseStore(delegate); + EXPECT_EQ(status, OK); + ASSERT_EQ(delegate, nullptr); +} + +/** + * @tc.name: RelationalStoreTest002 + * @tc.desc: Test open store with invalid path or store ID + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest002, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare db file + * @tc.expected: step1. Return OK. + */ + sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, NORMAL_CREATE_TABLE_SQL), SQLITE_OK); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); + + /** + * @tc.steps:step2. Test open store with invalid path or store ID + * @tc.expected: step2. open store failed. + */ + RelationalStoreDelegate *delegate = nullptr; + + // test open store with path not exist + DBStatus status = g_mgr.OpenStore(g_dbDir + "tmp/" + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate); + EXPECT_NE(status, OK); + ASSERT_EQ(delegate, nullptr); + + // test open store with empty store_id + status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, {}, {}, delegate); + EXPECT_NE(status, OK); + ASSERT_EQ(delegate, nullptr); + + // test open store with path has invalid character + status = g_mgr.OpenStore(g_dbDir + "t&m$p/" + STORE_ID + DB_SUFFIX, {}, {}, delegate); + EXPECT_NE(status, OK); + ASSERT_EQ(delegate, nullptr); + + // test open store with store_id has invalid character + status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, "Relation@al_S$tore_ID", {}, delegate); + EXPECT_NE(status, OK); + ASSERT_EQ(delegate, nullptr); + + // test open store with store_id length over MAX_STORE_ID_LENGTH + status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, + std::string(DBConstant::MAX_STORE_ID_LENGTH + 1, 'a'), {}, delegate); + EXPECT_NE(status, OK); + ASSERT_EQ(delegate, nullptr); +} + +/** + * @tc.name: RelationalStoreTest003 + * @tc.desc: Test open store with journal_mode is not WAL + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest003, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare db file with string is not WAL + * @tc.expected: step1. Return OK. + */ + sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=PRESIST;"), SQLITE_OK); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, NORMAL_CREATE_TABLE_SQL), SQLITE_OK); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); + + /** + * @tc.steps:step2. Test open store + * @tc.expected: step2. Open store failed. + */ + RelationalStoreDelegate *delegate = nullptr; + + // test open store with journal mode is not WAL + DBStatus status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate); + EXPECT_NE(status, OK); + ASSERT_EQ(delegate, nullptr); +} + +/** + * @tc.name: RelationalStoreTest004 + * @tc.desc: Test create distributed table with over limit + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest004, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare db file with multiple tables + * @tc.expected: step1. Return OK. + */ + sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); + const int tableCount = DBConstant::MAX_DISTRIBUTED_TABLE_COUNT + 10; // 10: additional size for test abnormal scene + for (int i=0; iCreateDistributedTable("TEST_" + std::to_string(i)), OK); + } else { + EXPECT_NE(delegate->CreateDistributedTable("TEST_" + std::to_string(i)), OK); + } + } + + /** + * @tc.steps:step3. Close store + * @tc.expected: step3. Return OK. + */ + status = g_mgr.CloseStore(delegate); + EXPECT_EQ(status, OK); + ASSERT_EQ(delegate, nullptr); +} + +/** + * @tc.name: RelationalStoreTest005 + * @tc.desc: Test create distributed table with invalid table name + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest005, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare db file + * @tc.expected: step1. Return OK. + */ + sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, NORMAL_CREATE_TABLE_SQL), SQLITE_OK); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); + + /** + * @tc.steps:step2. Open store + * @tc.expected: step2. return OK + */ + RelationalStoreDelegate *delegate = nullptr; + DBStatus status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate); + EXPECT_EQ(status, OK); + ASSERT_NE(delegate, nullptr); + + /** + * @tc.steps:step3. Create distirbuted table with invald table name + * @tc.expected: step3. Create distributed table failed. + */ + EXPECT_NE(delegate->CreateDistributedTable(DBConstant::SYSTEM_TABLE_PREFIX + "_tmp"), OK); + + /** + * @tc.steps:step4. Close store + * @tc.expected: step4. Return OK. + */ + status = g_mgr.CloseStore(delegate); + EXPECT_EQ(status, OK); + ASSERT_EQ(delegate, nullptr); +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp index f0e0baa1f..3e7b63492 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -192,7 +192,6 @@ void DistributedDBRelationalGetDataTest::TearDown(void) { if (g_delegate != nullptr) { EXPECT_EQ(g_mgr.CloseStore(g_delegate), DBStatus::OK); - EXPECT_EQ(g_mgr.DeleteStore(g_storePath), DBStatus::OK); g_delegate = nullptr; } if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index 40584d5fa..8ec7605e7 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -269,10 +269,6 @@ void DistributedDBRelationalVerP2PSyncTest::TearDown(void) LOGD("CloseStore Start"); ASSERT_EQ(g_mgr.CloseStore(g_kvDelegatePtr), OK); g_kvDelegatePtr = nullptr; - LOGD("DeleteStore Start"); - DBStatus status = g_mgr.DeleteStore(g_dbDir); - LOGD("delete kv store status %d", status); - ASSERT_TRUE(status == OK); } if (g_deviceB != nullptr) { delete g_deviceB; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp index abad85f7a..42b703606 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp @@ -292,5 +292,16 @@ int VirtualRelationalVerSyncDBInterface::GetAllSyncData(const std::string &table data = syncData_[tableName]; return E_OK; } + +int VirtualRelationalVerSyncDBInterface::CreateDistributedDeviceTable(const std::string &device, + const RelationalSyncStrategy &syncStrategy) +{ + return -E_NOT_SUPPORT; +} + +int VirtualRelationalVerSyncDBInterface::RegisterSchemaChangedCallback(const std::function &onSchemaChanged) +{ + return -E_NOT_SUPPORT; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h index a95fd5c83..318611909 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h @@ -87,6 +87,10 @@ public: return E_OK; } + int CreateDistributedDeviceTable(const std::string &device, const RelationalSyncStrategy &syncStrategy) override; + + int RegisterSchemaChangedCallback(const std::function &onSchemaChanged) override; + private: mutable std::map, std::vector> metadata_; -- Gitee From 6d62634d5584391a3313c331c2d31c5364ae6899 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 10 Jan 2022 19:20:56 +0800 Subject: [PATCH 03/38] Fix reviews Signed-off-by: lianhuix --- .../distributeddb/common/src/db_common.cpp | 3 +- .../relational/relational_schema_object.cpp | 36 ++++++++++++------- .../relational/relational_store_manager.h | 2 +- .../relational/relational_store_manager.cpp | 2 +- .../relational/relational_sync_able_storage.h | 1 + .../src/relational_sync_able_storage.cpp | 2 ++ .../relational/sqlite_relational_store.cpp | 10 +++--- ...single_ver_relational_storage_executor.cpp | 10 +++--- ...stributeddb_interfaces_relational_test.cpp | 9 ++--- 9 files changed, 43 insertions(+), 32 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp index 3ffe156b8..81a52ea55 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp @@ -314,6 +314,7 @@ std::string DBCommon::StringMasking(const std::string &oriStr, size_t remain) std::string DBCommon::GetDistributedTableName(const std::string &device, const std::string &tableName) { - return DBConstant::RELATIONAL_PREFIX + tableName + "_" + DBCommon::TransferHashString(device); + std::string deviceHashHex = DBCommon::TransferStringToHex(DBCommon::TransferHashString(device)); + return DBConstant::RELATIONAL_PREFIX + tableName + "_" + deviceHashHex; } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index 08fc8ac7d..0e9717662 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -98,8 +98,9 @@ std::string FieldInfo::ToAttributeString() const { std::string attrStr = "\"" + fieldName_ + "\": {"; attrStr += "\"TYPE\":\"" + dataType_ + "\","; - attrStr += "\"NOT_NULL\":" + std::string(isNotNull_ ? "true" : "false") + ","; + attrStr += "\"NOT_NULL\":" + std::string(isNotNull_ ? "true" : "false"); if (hasDefaultValue_) { + attrStr += ","; attrStr += "\"DEFAULT\":\"" + defaultValue_ + "\""; } attrStr += "}"; @@ -219,14 +220,14 @@ void TableInfo::AddUniqueDefineString(std::string &attrStr) const } attrStr += R"("UNIQUE": [)"; for (auto itUniqueDefine = uniqueDefines_.begin(); itUniqueDefine != uniqueDefines_.end(); ++itUniqueDefine) { - attrStr += "\""; + attrStr += "[\""; for (auto itField = (*itUniqueDefine).begin(); itField != (*itUniqueDefine).end(); ++itField) { attrStr += *itField; if (itField != (*itUniqueDefine).end() - 1) { - attrStr += ", "; + attrStr += "\",\""; } } - attrStr += "\""; + attrStr += "\"]"; if (itUniqueDefine != uniqueDefines_.end() - 1) { attrStr += ","; } @@ -241,14 +242,14 @@ void TableInfo::AddIndexDefineString(std::string &attrStr) const } attrStr += R"("INDEX": {)"; for (auto itIndexDefine = indexDefines_.begin(); itIndexDefine != indexDefines_.end(); ++itIndexDefine) { - attrStr += "\"" + (*itIndexDefine).first + "\": \""; + attrStr += "\"" + (*itIndexDefine).first + "\": [\""; for (auto itField = itIndexDefine->second.begin(); itField != itIndexDefine->second.end(); ++itField) { attrStr += *itField; if (itField != itIndexDefine->second.end() - 1) { - attrStr += ","; + attrStr += "\",\""; } } - attrStr += "\""; + attrStr += "\"]"; if (itIndexDefine != std::prev(indexDefines_.end(), 1)) { attrStr += ","; } @@ -418,8 +419,11 @@ void RelationalSchemaObject::GenerateSchemaString() schemaString_ += R"("SCHEMA_VERSION":"2.0",)"; schemaString_ += R"("SCHEMA_TYPE":"RELATIVE",)"; schemaString_ += R"("TABLES":[)"; - for (const auto &it : tables_) { - schemaString_ += it.second.ToTableInfoString() + ","; + for (auto it = tables_.begin(); it != tables_.end(); it++) { + if (it != tables_.begin()) { + schemaString_ += ","; + } + schemaString_ += it->second.ToTableInfoString(); } schemaString_ += R"(])"; schemaString_ += "}"; @@ -428,14 +432,14 @@ void RelationalSchemaObject::GenerateSchemaString() void RelationalSchemaObject::AddRelationalTable(const TableInfo &tb) { tables_[tb.GetTableName()] = tb; - schemaString_ = {}; + GenerateSchemaString(); } void RelationalSchemaObject::RemoveRelationalTable(const std::string &tableName) { tables_.erase(tableName); - schemaString_ = {}; + GenerateSchemaString(); } const std::map &RelationalSchemaObject::GetTables() const @@ -691,9 +695,12 @@ int RelationalSchemaObject::ParseCheckTableAutoInc(const JsonObject &inJsonObjec int RelationalSchemaObject::ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable) { + if (!inJsonObject.IsFieldPathExist(FieldPath {"UNIQUE"})) { // UNIQUE is not necessary + return E_OK; + } std::vector uniqueArray; int errCode = inJsonObject.GetArrayContentOfStringOrStringArray(FieldPath {"UNIQUE"}, uniqueArray); - if (errCode != E_OK && errCode != -E_INVALID_PATH) { // UNIQUE is not necessary + if (errCode != E_OK) { LOGE("[RelationalSchema][Parse] Get unique array failed: %d.", errCode); return -E_SCHEMA_PARSE_FAIL; } @@ -713,9 +720,12 @@ int RelationalSchemaObject::ParseCheckTablePrimaryKey(const JsonObject &inJsonOb int RelationalSchemaObject::ParseCheckTableIndex(const JsonObject &inJsonObject, TableInfo &resultTable) { + if (!inJsonObject.IsFieldPathExist(FieldPath {"INDEX"})) { // INDEX is not necessary + return E_OK; + } std::map tableFields; int errCode = inJsonObject.GetSubFieldPathAndType(FieldPath {"INDEX"}, tableFields); - if (errCode != E_OK && errCode != -E_INVALID_PATH) { // INDEX is not necessary + if (errCode != E_OK) { LOGE("[RelationalSchema][Parse] Get schema TABLES INDEX failed: %d.", errCode); return -E_SCHEMA_PARSE_FAIL; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 53c5d64c4..9ea601bbc 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -40,7 +40,7 @@ public: DB_API DBStatus OpenStore(const std::string &path, const std::string &storeId, const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate); - DB_API DBStatus CloseStore(RelationalStoreDelegate *&store); + DB_API DBStatus CloseStore(RelationalStoreDelegate *store); DB_API static void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 5fcf4916d..6ed6056a4 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -96,7 +96,7 @@ DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const return OK; } -DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *&store) +DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *store) { if (store == nullptr) { return INVALID_ARGS; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index 2d16239f2..9c110df5c 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -129,6 +129,7 @@ private: mutable std::mutex maxTimeStampMutex_; std::function onSchemaChanged_; + mutable std::mutex onSchemaChangedMutex_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index 883477569..b34d6459c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -468,12 +468,14 @@ int RelationalSyncAbleStorage::CreateDistributedDeviceTable(const std::string &d int RelationalSyncAbleStorage::RegisterSchemaChangedCallback(const std::function &callback) { + std::lock_guard lock(onSchemaChangedMutex_); onSchemaChanged_ = callback; return E_OK; } void RelationalSyncAbleStorage::NotifySchemaChanged() { + std::lock_guard lock(onSchemaChangedMutex_); if (onSchemaChanged_) { LOGD("Notify relational schema was changed"); onSchemaChanged_(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 7bda3da7a..ea0b4f036 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -171,11 +171,13 @@ int SQLiteRelationalStore::CleanDistributedDeviceTable() errCode = handle->CkeckAndCleanDistributedTable(schema.GettableNames(), missingTables); if (errCode == E_OK) { errCode = handle->Commit(); - // Remove non-existent tables from the schema - for (const auto &tableName : missingTables) { - schema.RemoveRelationalTable(tableName); + if (errCode == E_OK) { + // Remove non-existent tables from the schema + for (const auto &tableName : missingTables) { + schema.RemoveRelationalTable(tableName); + } + properties_.SetSchema(schema); } - properties_.SetSchema(schema); } else { LOGE("Check distributed table failed. %d", errCode); (void)handle->Rollback(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 1042e3adf..4fb415550 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -427,8 +427,7 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const { - std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = DBCommon::GetDistributedTableName(devName, object.GetTableName()); + const std::string tableName = DBCommon::GetDistributedTableName(deviceName, object.GetTableName()); TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode == -E_NOT_FOUND) { @@ -522,8 +521,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem) { - std::string devName = DBCommon::TransferHashString(dataItem.dev); - const std::string tableName = DBCommon::GetDistributedTableName(devName, table_.GetTableName()); + const std::string tableName = DBCommon::GetDistributedTableName(dataItem.dev, table_.GetTableName()); std::string hashKey = std::string(dataItem.hashKey.begin(), dataItem.hashKey.end()); std::string sql = "DELETE FROM " + tableName + " WHERE calc_hash(" + table_.GetPrimaryKey() + ")=" + hashKey + ";"; sqlite3_stmt *stmt = nullptr; @@ -734,7 +732,7 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteDistributedDeviceTable(const std::string tablePattern = tableName.empty() ? "%" : tableName; std::string deviceTableName = DBConstant::RELATIONAL_PREFIX + tablePattern + "_" + decicePattern; - static const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE " + + const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE " + deviceTableName + ";"; sqlite3_stmt *stmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, checkSql, stmt); @@ -786,7 +784,7 @@ int SQLiteSingleVerRelationalStorageExecutor::CkeckAndCleanDistributedTable(cons if (tableNames.empty()) { return E_OK; } - static const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name='?';"; + const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name='?';"; sqlite3_stmt *stmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, checkSql, stmt); if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp index d350dc1cb..efb46e2b6 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp @@ -33,14 +33,15 @@ namespace { DistributedDB::RelationalStoreManager g_mgr(APP_ID, USER_ID); const std::string NORMAL_CREATE_TABLE_SQL = "CREATE TABLE IF NOT EXISTS sync_data(" \ - "key BLOB NOT NULL," \ + "key BLOB NOT NULL UNIQUE," \ "value BLOB," \ "timestamp INT NOT NULL," \ "flag INT NOT NULL," \ "device BLOB," \ "ori_device BLOB," \ "hash_key BLOB PRIMARY KEY NOT NULL," \ - "w_timestamp INT);" \ + "w_timestamp INT," \ + "UNIQUE(device, ori_device));" \ "CREATE INDEX key_index ON sync_data (key, flag);"; } @@ -110,7 +111,6 @@ HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest001, TestSize.L status = g_mgr.CloseStore(delegate); EXPECT_EQ(status, OK); - ASSERT_EQ(delegate, nullptr); /** * @tc.steps:step3. drop sync_data table @@ -131,7 +131,6 @@ HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest001, TestSize.L ASSERT_NE(delegate, nullptr); status = g_mgr.CloseStore(delegate); EXPECT_EQ(status, OK); - ASSERT_EQ(delegate, nullptr); } /** @@ -264,7 +263,6 @@ HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest004, TestSize.L */ status = g_mgr.CloseStore(delegate); EXPECT_EQ(status, OK); - ASSERT_EQ(delegate, nullptr); } /** @@ -307,5 +305,4 @@ HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest005, TestSize.L */ status = g_mgr.CloseStore(delegate); EXPECT_EQ(status, OK); - ASSERT_EQ(delegate, nullptr); } \ No newline at end of file -- Gitee From 7842d4b9c5048d25dd59faa3a9a6c2df0b32a7d3 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 18 Jan 2022 15:18:22 +0800 Subject: [PATCH 04/38] Fix reviews Signed-off-by: lianhuix --- .../sqlite/relational/sqlite_relational_store.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index ea0b4f036..fbf1a25a2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -148,7 +148,7 @@ int SQLiteRelationalStore::SaveLogTableVersionToMeta() if (errCode != E_OK) { LOGE("save log table version to meta table failed. %d", errCode); } - return E_OK; + return errCode; } int SQLiteRelationalStore::CleanDistributedDeviceTable() @@ -375,10 +375,12 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) ReleaseHandle(handle); return errCode; } - (void)handle->Commit(); - schema.AddRelationalTable(table); - properties_.SetSchema(schema); - storageEngine_->NotifySchemaChanged(); + errCode = handle->Commit(); + if (errCode == E_OK) { + schema.AddRelationalTable(table); + properties_.SetSchema(schema); + storageEngine_->NotifySchemaChanged(); + } ReleaseHandle(handle); return SaveSchemaToMeta(); -- Gitee From b6c88bee4e3d52978f43d6739b877de00e372d24 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 18 Jan 2022 16:40:16 +0800 Subject: [PATCH 05/38] Fix reviews. Signed-off-by: lianhuix --- .../common/include/relational/relational_schema_object.h | 2 +- .../common/src/relational/relational_schema_object.cpp | 2 +- .../storage/src/sqlite/relational/sqlite_relational_store.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index 75a5e8af1..1a1f6cf92 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -140,7 +140,7 @@ public: const std::map &GetTables() const; - std::vector GettableNames() const; + std::vector GetTableNames() const; TableInfo GetTable(const std::string& tableName) const; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index 0e9717662..b500194ad 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -447,7 +447,7 @@ const std::map &RelationalSchemaObject::GetTables() cons return tables_; } -std::vector RelationalSchemaObject::GettableNames() const +std::vector RelationalSchemaObject::GetTableNames() const { std::vector tableNames; for (const auto &it : tables_) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index fbf1a25a2..18b01a3de 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -168,7 +168,7 @@ int SQLiteRelationalStore::CleanDistributedDeviceTable() std::lock_guard lock(schemaMutex_); RelationalSchemaObject schema = properties_.GetSchema(); std::vector missingTables; - errCode = handle->CkeckAndCleanDistributedTable(schema.GettableNames(), missingTables); + errCode = handle->CkeckAndCleanDistributedTable(schema.GetTableNames(), missingTables); if (errCode == E_OK) { errCode = handle->Commit(); if (errCode == E_OK) { @@ -184,7 +184,7 @@ int SQLiteRelationalStore::CleanDistributedDeviceTable() } ReleaseHandle(handle); - return errCode; + return SaveSchemaToMeta(); } int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) -- Gitee From 4b4845d8a979db253ceea0def54dabefc00f0036 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Tue, 18 Jan 2022 10:32:07 +0800 Subject: [PATCH 06/38] Solve memory leak issue. Signed-off-by: lidwchn --- ...te_single_ver_relational_continue_token.cpp | 2 +- .../storage/src/sqlite/sqlite_utils.cpp | 2 +- .../distributeddb_relational_get_data_test.cpp | 18 +++++++++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp index 6e6192733..bc2150f84 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp @@ -19,7 +19,7 @@ namespace DistributedDB { SQLiteSingleVerRelationalContinueToken::SQLiteSingleVerRelationalContinueToken( const SyncTimeRange &timeRange, const QueryObject &object) - : isGettingDeletedData_(false), queryObj_(object), tableName_(object.GetTableName()), timeRange_(timeRange) + : isGettingDeletedData_(false), queryObj_(object), tableName_(queryObj_.GetTableName()), timeRange_(timeRange) {} bool SQLiteSingleVerRelationalContinueToken::CheckValid() const diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index b4f6efaff..e44a55727 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1411,7 +1411,7 @@ int SQLiteUtils::CloneIndexes(sqlite3 *db, const std::string &oriTableName, cons { std::string sql = "SELECT 'CREATE ' || CASE WHEN il.'unique' THEN 'UNIQUE ' ELSE '' END || 'INDEX ' || '" + - DBConstant::RELATIONAL_PREFIX + "' || il.name || ' ON ' || '" + newTableName + + newTableName + "_' || il.name || ' ON ' || '" + newTableName + "' || '(' || GROUP_CONCAT(ii.name) || ');' " "FROM sqlite_master AS m," "pragma_index_list(m.name) AS il," diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp index 3e7b63492..db783e812 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -45,6 +45,7 @@ string g_storeID = "dftStoreID"; const string g_tableName { "data" }; DistributedDB::RelationalStoreManager g_mgr(APP_ID, USER_ID); RelationalStoreDelegate *g_delegate = nullptr; +IRelationalStore *g_store = nullptr; void CreateDBAndTable() { @@ -117,8 +118,8 @@ int GetLogData(int key, uint64_t &flag, TimeStamp ×tamp, const DeviceID &de } END: - sqlite3_close(db); SQLiteUtils::ResetStatement(statement, true, errCode); + sqlite3_close(db); return errCode; } @@ -139,12 +140,12 @@ const RelationalSyncAbleStorage *GetRelationalStore() RelationalDBProperties properties; InitStoreProp(g_storePath, APP_ID, USER_ID, properties); int errCode = E_OK; - auto store = RelationalStoreInstance::GetDataBase(properties, errCode); - if (store == nullptr) { + g_store = RelationalStoreInstance::GetDataBase(properties, errCode); + if (g_store == nullptr) { LOGE("Get db failed:%d", errCode); return nullptr; } - return static_cast(store)->GetStorageEngine(); + return static_cast(g_store)->GetStorageEngine(); } int GetCount(sqlite3 *db, const string &sql, size_t &count) @@ -259,7 +260,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) * @tc.expected: Succeed and the count is right. */ auto store = GetRelationalStore(); - ASSERT_EQ(store, nullptr); + ASSERT_NE(store, nullptr); ContinueToken token = nullptr; QueryObject query(Query::Select(g_tableName)); std::vector entries; @@ -276,6 +277,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) EXPECT_TRUE(errCode == E_OK || errCode == -E_UNFINISHED); } EXPECT_EQ(count, RECORD_COUNT); + RefObject::DecObjRef(g_store); } /** @@ -305,7 +307,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) * @tc.expected: Get 70 records. */ auto store = GetRelationalStore(); - ASSERT_EQ(store, nullptr); + ASSERT_NE(store, nullptr); ContinueToken token = nullptr; const unsigned int LIMIT = 80; // limit as 80. const unsigned int OFFSET = 30; // offset as 30. @@ -318,6 +320,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) EXPECT_EQ(errCode, E_OK); EXPECT_EQ(token, nullptr); SingleVerKvEntry::Release(entries); + RefObject::DecObjRef(g_store); } /** @@ -373,7 +376,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Lev * @tc.expected: Succeed and the count is right. */ auto store = GetRelationalStore(); - ASSERT_EQ(store, nullptr); + ASSERT_NE(store, nullptr); ContinueToken token = nullptr; QueryObject query(Query::Select(tableName)); std::vector entries; @@ -410,5 +413,6 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Lev EXPECT_EQ(GetCount(db, sql, count), E_OK); EXPECT_EQ(count, 2UL); // The index count is 2. sqlite3_close(db); + RefObject::DecObjRef(g_store); } #endif \ No newline at end of file -- Gitee From 9dcfc44868726d063c390bbe0c0cd4a34ac02afd Mon Sep 17 00:00:00 2001 From: zqq Date: Wed, 5 Jan 2022 16:36:01 +0800 Subject: [PATCH 07/38] Support rdb autoLaunch Signed-off-by: zqq --- .../common/include/auto_launch.h | 60 ++- .../common/include/runtime_context.h | 11 +- .../distributeddb/common/src/auto_launch.cpp | 383 +++++++++++++----- .../common/src/runtime_context_impl.cpp | 4 +- .../common/src/runtime_context_impl.h | 2 +- .../interfaces/include/data_value.h | 4 + .../src/kv_store_delegate_manager.cpp | 9 +- .../interfaces/src/relational/data_value.cpp | 89 ++-- .../relational/relational_store_manager.cpp | 1 + .../relational/relational_sync_able_storage.h | 1 + .../storage/include/sync_generic_interface.h | 10 + .../src/relational_sync_able_storage.cpp | 8 + .../src/single_ver_sync_task_context.cpp | 4 +- .../common/distributeddb_auto_launch_test.cpp | 22 +- ...ributeddb_relational_ver_p2p_sync_test.cpp | 380 ++++++++++++++--- .../syncer/distributeddb_time_sync_test.cpp | 1 + .../common/syncer/generic_virtual_device.cpp | 8 +- .../common/syncer/generic_virtual_device.h | 1 + .../syncer/relational_virtual_device.cpp | 10 +- .../common/syncer/relational_virtual_device.h | 8 +- ...rtual_relational_ver_sync_db_interface.cpp | 70 ++-- ...virtual_relational_ver_sync_db_interface.h | 15 +- 22 files changed, 840 insertions(+), 261 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h index 53bed5681..a275a5e44 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h @@ -19,12 +19,15 @@ #include #include #include -#include "types_export.h" -#include "kv_store_observer.h" -#include "kvdb_properties.h" +#include "auto_launch_export.h" +#include "db_properties.h" #include "ikvdb_connection.h" #include "icommunicator_aggregator.h" -#include "auto_launch_export.h" +#include "kv_store_observer.h" +#include "kvdb_properties.h" +#include "types_export.h" +#include "relational_store_connection.h" +#include "relationaldb_properties.h" namespace DistributedDB { enum class AutoLaunchItemState { @@ -35,19 +38,26 @@ enum class AutoLaunchItemState { IDLE, }; +enum class DBType { + DB_KV = 0, + DB_RELATION, + DB_INVALID, +}; + struct AutoLaunchItem { - KvDBProperties properties; + std::shared_ptr propertiesPtr; AutoLaunchNotifier notifier; KvStoreObserver *observer = nullptr; int conflictType = 0; KvStoreNbConflictNotifier conflictNotifier; - IKvDBConnection *conn = nullptr; + void *conn = nullptr; KvDBObserverHandle *observerHandle = nullptr; bool isWriteOpenNotifiered = false; AutoLaunchItemState state = AutoLaunchItemState::UN_INITIAL; bool isDisable = false; bool inObserver = false; bool isAutoSync = true; + DBType type = DBType::DB_INVALID; }; class AutoLaunch { @@ -67,9 +77,10 @@ public: void GetAutoLaunchSyncDevices(const std::string &identifier, std::vector &devices) const; - void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback); + void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback, DBType type); - static int GetAutoLaunchProperties(const AutoLaunchParam ¶m, KvDBProperties &properties); + static int GetAutoLaunchProperties(const AutoLaunchParam ¶m, const DBType &openType, + std::shared_ptr &propertiesPtr); private: @@ -77,13 +88,13 @@ private: int GetConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std::string &identifier); - IKvDBConnection *GetOneConnection(const KvDBProperties &properties, int &errCode); + static int OpenOneConnection(AutoLaunchItem &autoLaunchItem); // we will return errCode, if errCode != E_OK - int CloseConnectionStrict(AutoLaunchItem &autoLaunchItem); + static int CloseConnectionStrict(AutoLaunchItem &autoLaunchItem); // before ReleaseDatabaseConnection, if errCode != E_OK, we not return, we try close more - void TryCloseConnection(AutoLaunchItem &autoLaunchItem); + static void TryCloseConnection(AutoLaunchItem &autoLaunchItem); int RegisterObserverAndLifeCycleCallback(AutoLaunchItem &autoLaunchItem, const std::string &identifier, bool isExt); @@ -104,7 +115,7 @@ private: void ReceiveUnknownIdentifierCallBackTask(const std::string &identifier); - void CloseNotifier(const AutoLaunchItem &autoLaunchItem); + static void CloseNotifier(const AutoLaunchItem &autoLaunchItem); void ConnectionLifeCycleCallback(const std::string &identifier); @@ -122,7 +133,28 @@ private: void ExtConnectionLifeCycleCallbackTask(const std::string &identifier); - int SetConflictNotifier(IKvDBConnection *conn, int conflictType, const KvStoreNbConflictNotifier ¬ifier); + static int SetConflictNotifier(AutoLaunchItem &autoLaunchItem); + + int ExtAutoLaunchRequestCallBack(const std::string &identifier, AutoLaunchParam ¶m, DBType &openType); + + static int GetAutoLaunchKVProperties(const AutoLaunchParam ¶m, + const std::shared_ptr &propertiesPtr); + + static int GetAutoLaunchRelationProperties(const AutoLaunchParam ¶m, + const std::shared_ptr &propertiesPtr); + + static int OpenOneKvConnection(AutoLaunchItem &autoLaunchItem); + + static int OpenOneRelationalConnection(AutoLaunchItem &autoLaunchItem); + + int RegisterLifeCycleCallback(AutoLaunchItem &autoLaunchItem, const std::string &identifier, + bool isExt); + + static int PragmaAutoSync(AutoLaunchItem &autoLaunchItem); + + static void TryCloseKvConnection(AutoLaunchItem &autoLaunchItem); + + static void TryCloseRelationConnection(AutoLaunchItem &autoLaunchItem); mutable std::mutex dataLock_; mutable std::mutex communicatorLock_; @@ -132,7 +164,7 @@ private: std::condition_variable cv_; std::mutex extLock_; - AutoLaunchRequestCallback autoLaunchRequestCallback_; + std::map autoLaunchRequestCallbackMap_; std::map extItemMap_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h index 5d704f94b..cb20a7951 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h @@ -20,14 +20,15 @@ #include #include -#include "macro_utils.h" -#include "notification_chain.h" +#include "auto_launch.h" +#include "auto_launch_export.h" #include "icommunicator_aggregator.h" #include "iprocess_system_api_adapter.h" -#include "types_export.h" #include "kv_store_observer.h" #include "kvdb_properties.h" -#include "auto_launch_export.h" +#include "macro_utils.h" +#include "notification_chain.h" +#include "types_export.h" namespace DistributedDB { using TimerId = uint64_t; @@ -85,7 +86,7 @@ public: virtual void GetAutoLaunchSyncDevices(const std::string &identifier, std::vector &devices) const = 0; - virtual void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) = 0; + virtual void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback, DBType type) = 0; virtual NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errCode) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp index b55ad13d8..b39122471 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp @@ -25,6 +25,7 @@ #include "kvdb_pragma.h" #include "log_print.h" #include "param_check_utils.h" +#include "relational_store_instance.h" #include "runtime_context.h" #include "semaphore_utils.h" #include "sync_able_kvdb_connection.h" @@ -136,8 +137,10 @@ int AutoLaunch::EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLa { LOGI("[AutoLaunch] EnableKvStoreAutoLaunch"); std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); - AutoLaunchItem autoLaunchItem { properties, notifier, option.observer, option.conflictType, option.notifier }; + std::shared_ptr ptr = std::make_shared(properties); + AutoLaunchItem autoLaunchItem { ptr, notifier, option.observer, option.conflictType, option.notifier }; autoLaunchItem.isAutoSync = option.isAutoSync; + autoLaunchItem.type = DBType::DB_KV; int errCode = EnableKvStoreAutoLaunchParmCheck(autoLaunchItem, identifier); if (errCode != E_OK) { LOGE("[AutoLaunch] EnableKvStoreAutoLaunch failed errCode:%d", errCode); @@ -156,7 +159,9 @@ int AutoLaunch::GetConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std: { LOGI("[AutoLaunch] GetConnectionInEnable"); int errCode; - autoLaunchItem.conn = KvDBManager::GetDatabaseConnection(autoLaunchItem.properties, errCode, false); + std::shared_ptr properties = + std::static_pointer_cast(autoLaunchItem.propertiesPtr); + autoLaunchItem.conn = KvDBManager::GetDatabaseConnection(*properties, errCode, false); if (errCode == -E_ALREADY_OPENED) { LOGI("[AutoLaunch] GetConnectionInEnable user already getkvstore by self"); std::lock_guard autoLock(dataLock_); @@ -176,7 +181,8 @@ int AutoLaunch::GetConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std: } if (isEmpty) { LOGI("[AutoLaunch] GetConnectionInEnable no online device, ReleaseDatabaseConnection"); - errCode = KvDBManager::ReleaseDatabaseConnection(autoLaunchItem.conn); + IKvDBConnection *kvConn = static_cast(autoLaunchItem.conn); + errCode = KvDBManager::ReleaseDatabaseConnection(kvConn); if (errCode != E_OK) { LOGE("[AutoLaunch] GetConnectionInEnable ReleaseDatabaseConnection failed errCode:%d", errCode); std::lock_guard autoLock(dataLock_); @@ -211,20 +217,21 @@ int AutoLaunch::CloseConnectionStrict(AutoLaunchItem &autoLaunchItem) LOGI("[AutoLaunch] CloseConnectionStrict conn is nullptr, do nothing"); return E_OK; } - int errCode = autoLaunchItem.conn->RegisterLifeCycleCallback(nullptr); + IKvDBConnection *kvConn = static_cast(autoLaunchItem.conn); + int errCode = kvConn->RegisterLifeCycleCallback(nullptr); if (errCode != E_OK) { LOGE("[AutoLaunch] CloseConnectionStrict RegisterLifeCycleCallback failed errCode:%d", errCode); return errCode; } if (autoLaunchItem.observerHandle != nullptr) { - errCode = autoLaunchItem.conn->UnRegisterObserver(autoLaunchItem.observerHandle); + errCode = kvConn->UnRegisterObserver(autoLaunchItem.observerHandle); if (errCode != E_OK) { LOGE("[AutoLaunch] CloseConnectionStrict UnRegisterObserver failed errCode:%d", errCode); return errCode; } autoLaunchItem.observerHandle = nullptr; } - errCode = KvDBManager::ReleaseDatabaseConnection(autoLaunchItem.conn); + errCode = KvDBManager::ReleaseDatabaseConnection(kvConn); if (errCode != E_OK) { LOGE("[AutoLaunch] CloseConnectionStrict ReleaseDatabaseConnection failed errCode:%d", errCode); } @@ -235,24 +242,16 @@ int AutoLaunch::CloseConnectionStrict(AutoLaunchItem &autoLaunchItem) void AutoLaunch::TryCloseConnection(AutoLaunchItem &autoLaunchItem) { LOGI("[AutoLaunch] TryCloseConnection"); - if (autoLaunchItem.conn == nullptr) { - LOGI("[AutoLaunch] TryCloseConnection conn is nullptr, do nothing"); - return; - } - int errCode = autoLaunchItem.conn->RegisterLifeCycleCallback(nullptr); - if (errCode != E_OK) { - LOGE("[AutoLaunch] TryCloseConnection RegisterLifeCycleCallback failed errCode:%d", errCode); - } - if (autoLaunchItem.observerHandle != nullptr) { - errCode = autoLaunchItem.conn->UnRegisterObserver(autoLaunchItem.observerHandle); - if (errCode != E_OK) { - LOGE("[AutoLaunch] TryCloseConnection UnRegisterObserver failed errCode:%d", errCode); - } - autoLaunchItem.observerHandle = nullptr; - } - errCode = KvDBManager::ReleaseDatabaseConnection(autoLaunchItem.conn); - if (errCode != E_OK) { - LOGE("[AutoLaunch] TryCloseConnection ReleaseDatabaseConnection failed errCode:%d", errCode); + switch (autoLaunchItem.type) { + case DBType::DB_KV: + TryCloseKvConnection(autoLaunchItem); + break; + case DBType::DB_RELATION: + TryCloseRelationConnection(autoLaunchItem); + break; + default: + LOGD("[AutoLaunch] Unknown type[%d] when try to close connection", autoLaunchItem.type); + break; } } @@ -264,39 +263,31 @@ int AutoLaunch::RegisterObserverAndLifeCycleCallback(AutoLaunchItem &autoLaunchI return errCode; } LOGI("[AutoLaunch] RegisterObserver ok"); - if (isExt) { - errCode = autoLaunchItem.conn->RegisterLifeCycleCallback(std::bind( - &AutoLaunch::ExtConnectionLifeCycleCallback, this, std::placeholders::_1)); - } else { - errCode = autoLaunchItem.conn->RegisterLifeCycleCallback(std::bind(&AutoLaunch::ConnectionLifeCycleCallback, - this, std::placeholders::_1)); - } + + errCode = RegisterLifeCycleCallback(autoLaunchItem, identifier, isExt); if (errCode != E_OK) { LOGE("[AutoLaunch] RegisterLifeCycleCallback failed, errCode:%d", errCode); return errCode; } LOGI("[AutoLaunch] RegisterLifeCycleCallback ok"); - errCode = SetConflictNotifier(autoLaunchItem.conn, autoLaunchItem.conflictType, autoLaunchItem.conflictNotifier); + errCode = SetConflictNotifier(autoLaunchItem); if (errCode != E_OK) { LOGE("[AutoLaunch] SetConflictNotifier failed, errCode:%d", errCode); return errCode; } - bool enAutoSync = autoLaunchItem.isAutoSync; - errCode = static_cast(autoLaunchItem.conn)->Pragma(PRAGMA_AUTO_SYNC, - static_cast(&enAutoSync)); - if (errCode != E_OK) { - LOGE("[AutoLaunch] PRAGMA_AUTO_SYNC failed, errCode:%d", errCode); - return errCode; - } - LOGI("[AutoLaunch] set PRAGMA_AUTO_SYNC ok, enAutoSync=%d", enAutoSync); - return errCode; + return PragmaAutoSync(autoLaunchItem); } int AutoLaunch::RegisterObserver(AutoLaunchItem &autoLaunchItem, const std::string &identifier, bool isExt) { LOGI("[AutoLaunch] RegisterObserver"); + if (autoLaunchItem.type != DBType::DB_KV) { + LOGD("[AutoLaunch] Current Type[%d] Not Support Observer", autoLaunchItem.type); + return E_OK; + } + if (autoLaunchItem.conn == nullptr) { LOGE("[AutoLaunch] autoLaunchItem.conn is nullptr"); return -E_INTERNAL_ERROR; @@ -304,11 +295,12 @@ int AutoLaunch::RegisterObserver(AutoLaunchItem &autoLaunchItem, const std::stri int errCode; Key key; KvDBObserverHandle *observerHandle = nullptr; + IKvDBConnection *kvConn = static_cast(autoLaunchItem.conn); if (isExt) { - observerHandle = autoLaunchItem.conn->RegisterObserver(OBSERVER_CHANGES_FOREIGN, key, + observerHandle = kvConn->RegisterObserver(OBSERVER_CHANGES_FOREIGN, key, std::bind(&AutoLaunch::ExtObserverFunc, this, std::placeholders::_1, identifier), errCode); } else { - observerHandle = autoLaunchItem.conn->RegisterObserver(OBSERVER_CHANGES_FOREIGN, key, + observerHandle = kvConn->RegisterObserver(OBSERVER_CHANGES_FOREIGN, key, std::bind(&AutoLaunch::ObserverFunc, this, std::placeholders::_1, identifier), errCode); } @@ -341,9 +333,12 @@ void AutoLaunch::ObserverFunc(const KvDBCommitNotifyData ¬ifyData, const std: autoLaunchItem.observer = autoLaunchItemMap_[identifier].observer; autoLaunchItem.isWriteOpenNotifiered = autoLaunchItemMap_[identifier].isWriteOpenNotifiered; autoLaunchItem.notifier = autoLaunchItemMap_[identifier].notifier; - userId = autoLaunchItemMap_[identifier].properties.GetStringProp(KvDBProperties::USER_ID, ""); - appId = autoLaunchItemMap_[identifier].properties.GetStringProp(KvDBProperties::APP_ID, ""); - storeId = autoLaunchItemMap_[identifier].properties.GetStringProp(KvDBProperties::STORE_ID, ""); + + std::shared_ptr properties = + std::static_pointer_cast(autoLaunchItemMap_[identifier].propertiesPtr); + userId = properties->GetStringProp(KvDBProperties::USER_ID, ""); + appId = properties->GetStringProp(KvDBProperties::APP_ID, ""); + storeId = properties->GetStringProp(KvDBProperties::STORE_ID, ""); } if (autoLaunchItem.observer != nullptr) { LOGI("[AutoLaunch] do user observer"); @@ -440,9 +435,11 @@ void AutoLaunch::GetAutoLaunchSyncDevices(const std::string &identifier, std::ve void AutoLaunch::CloseNotifier(const AutoLaunchItem &autoLaunchItem) { if (autoLaunchItem.notifier) { - std::string userId = autoLaunchItem.properties.GetStringProp(KvDBProperties::USER_ID, ""); - std::string appId = autoLaunchItem.properties.GetStringProp(KvDBProperties::APP_ID, ""); - std::string storeId = autoLaunchItem.properties.GetStringProp(KvDBProperties::STORE_ID, ""); + std::shared_ptr properties = + std::static_pointer_cast(autoLaunchItem.propertiesPtr); + std::string userId = properties->GetStringProp(KvDBProperties::USER_ID, ""); + std::string appId = properties->GetStringProp(KvDBProperties::APP_ID, ""); + std::string storeId = properties->GetStringProp(KvDBProperties::STORE_ID, ""); LOGI("[AutoLaunch] CloseNotifier do autoLaunchItem.notifier"); autoLaunchItem.notifier(userId, appId, storeId, AutoLaunchStatus::WRITE_CLOSED); LOGI("[AutoLaunch] CloseNotifier do autoLaunchItem.notifier finished"); @@ -499,16 +496,17 @@ void AutoLaunch::ConnectionLifeCycleCallback(const std::string &identifier) } } -IKvDBConnection *AutoLaunch::GetOneConnection(const KvDBProperties &properties, int &errCode) +int AutoLaunch::OpenOneConnection(AutoLaunchItem &autoLaunchItem) { LOGI("[AutoLaunch] GetOneConnection"); - IKvDBConnection *conn = KvDBManager::GetDatabaseConnection(properties, errCode, false); - if (errCode == -E_ALREADY_OPENED) { - LOGI("[AutoLaunch] GetOneConnection user already getkvstore by self"); - } else if (conn == nullptr) { - LOGE("[AutoLaunch] GetOneConnection GetDatabaseConnection failed errCode:%d", errCode); + switch (autoLaunchItem.type) { + case DBType::DB_KV: + return OpenOneKvConnection(autoLaunchItem); + case DBType::DB_RELATION: + return OpenOneRelationalConnection(autoLaunchItem); + default: + return -E_INVALID_ARGS; } - return conn; } void AutoLaunch::OnlineCallBack(const std::string &device, bool isConnect) @@ -570,8 +568,7 @@ void AutoLaunch::GetConnInDoOpenMap(std::map &doOpe SemaphoreUtils sema(1 - doOpenMap.size()); for (auto &iter : doOpenMap) { int errCode = RuntimeContext::GetInstance()->ScheduleTask([&sema, &iter, this] { - int ret; - iter.second.conn = GetOneConnection(iter.second.properties, ret); + int ret = OpenOneConnection(iter.second); LOGI("[AutoLaunch] GetConnInDoOpenMap GetOneConnection errCode:%d\n", ret); if (iter.second.conn == nullptr) { sema.SendSemaphore(); @@ -623,8 +620,7 @@ void AutoLaunch::ReceiveUnknownIdentifierCallBackTask(const std::string &identif std::lock_guard autoLock(dataLock_); autoLaunchItem = autoLaunchItemMap_[identifier]; } - int errCode; - autoLaunchItem.conn = GetOneConnection(autoLaunchItem.properties, errCode); + int errCode = OpenOneConnection(autoLaunchItem); LOGI("[AutoLaunch] ReceiveUnknownIdentifierCallBack GetOneConnection errCode:%d\n", errCode); if (autoLaunchItem.conn == nullptr) { std::lock_guard autoLock(dataLock_); @@ -691,30 +687,28 @@ EXT: return AutoLaunchExt(identifier); } -void AutoLaunch::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) +void AutoLaunch::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback, DBType type) { - LOGI("[AutoLaunch] SetAutoLaunchRequestCallback"); + LOGI("[AutoLaunch] SetAutoLaunchRequestCallback type[%d]", type); std::lock_guard lock(extLock_); - autoLaunchRequestCallback_ = callback; + if (callback) { + autoLaunchRequestCallbackMap_[type] = callback; + } else if (autoLaunchRequestCallbackMap_.find(type) != autoLaunchRequestCallbackMap_.end()) { + autoLaunchRequestCallbackMap_.erase(type); + } } int AutoLaunch::AutoLaunchExt(const std::string &identifier) { AutoLaunchParam param; - { - std::lock_guard lock(extLock_); - if (!autoLaunchRequestCallback_) { - LOGI("[AutoLaunch] autoLaunchRequestCallback_ is nullptr"); - return -E_NOT_FOUND; // not E_OK is ok for communicator - } - bool needOpen = autoLaunchRequestCallback_(identifier, param); - if (!needOpen) { - LOGI("[AutoLaunch] autoLaunchRequestCallback_ is not need open"); - return -E_NOT_FOUND; // not E_OK is ok for communicator - } + DBType openType = DBType::DB_INVALID; + int errCode = ExtAutoLaunchRequestCallBack(identifier, param, openType); + if (errCode != E_OK) { + return errCode; // not E_OK is ok for communicator } - KvDBProperties properties; - int errCode = AutoLaunch::GetAutoLaunchProperties(param, properties); + + std::shared_ptr ptr; + errCode = AutoLaunch::GetAutoLaunchProperties(param, openType, ptr); if (errCode != E_OK) { LOGE("[AutoLaunch] AutoLaunchExt param check fail errCode:%d", errCode); if (!param.notifier) { @@ -728,9 +722,10 @@ int AutoLaunch::AutoLaunchExt(const std::string &identifier) } return errCode; } - AutoLaunchItem autoLaunchItem{properties, param.notifier, param.option.observer, param.option.conflictType, + AutoLaunchItem autoLaunchItem{ptr, param.notifier, param.option.observer, param.option.conflictType, param.option.notifier}; autoLaunchItem.isAutoSync = param.option.isAutoSync; + autoLaunchItem.type = openType; errCode = RuntimeContext::GetInstance()->ScheduleTask(std::bind(&AutoLaunch::AutoLaunchExtTask, this, identifier, autoLaunchItem)); if (errCode != E_OK) { @@ -749,8 +744,7 @@ void AutoLaunch::AutoLaunchExtTask(const std::string identifier, AutoLaunchItem } extItemMap_[identifier] = autoLaunchItem; } - int errCode; - autoLaunchItem.conn = GetOneConnection(autoLaunchItem.properties, errCode); + int errCode = OpenOneConnection(autoLaunchItem); LOGI("[AutoLaunch] AutoLaunchExtTask GetOneConnection errCode:%d", errCode); if (autoLaunchItem.conn == nullptr) { std::lock_guard autoLock(extLock_); @@ -803,9 +797,9 @@ void AutoLaunch::ExtObserverFunc(const KvDBCommitNotifyData ¬ifyData, const s } } - std::string userId = autoLaunchItem.properties.GetStringProp(KvDBProperties::USER_ID, ""); - std::string appId = autoLaunchItem.properties.GetStringProp(KvDBProperties::APP_ID, ""); - std::string storeId = autoLaunchItem.properties.GetStringProp(KvDBProperties::STORE_ID, ""); + std::string userId = autoLaunchItem.propertiesPtr->GetStringProp(KvDBProperties::USER_ID, ""); + std::string appId = autoLaunchItem.propertiesPtr->GetStringProp(KvDBProperties::APP_ID, ""); + std::string storeId = autoLaunchItem.propertiesPtr->GetStringProp(KvDBProperties::STORE_ID, ""); int retCode = RuntimeContext::GetInstance()->ScheduleTask([notifier, userId, appId, storeId] { LOGI("[AutoLaunch] ExtObserverFunc do user notifier WRITE_OPENED"); notifier(userId, appId, storeId, AutoLaunchStatus::WRITE_OPENED); @@ -849,18 +843,26 @@ void AutoLaunch::ExtConnectionLifeCycleCallbackTask(const std::string &identifie } } -int AutoLaunch::SetConflictNotifier(IKvDBConnection *conn, int conflictType, const KvStoreNbConflictNotifier ¬ifier) +int AutoLaunch::SetConflictNotifier(AutoLaunchItem &autoLaunchItem) { + if (autoLaunchItem.type != DBType::DB_KV) { + LOGD("[AutoLaunch] Current Type[%d] Not Support ConflictNotifier Now", autoLaunchItem.type); + return E_OK; + } + + IKvDBConnection *kvConn = static_cast(autoLaunchItem.conn); + int conflictType = autoLaunchItem.conflictType; + const KvStoreNbConflictNotifier ¬ifier = autoLaunchItem.conflictNotifier; if (conflictType == 0) { return E_OK; } int errCode; if (!notifier) { - errCode = conn->SetConflictNotifier(conflictType, nullptr); + errCode = kvConn->SetConflictNotifier(conflictType, nullptr); goto END; } - errCode = conn->SetConflictNotifier(conflictType, + errCode = kvConn->SetConflictNotifier(conflictType, [conflictType, notifier](const KvDBCommitNotifyData &data) { int resultCode; const std::list entries = data.GetCommitConflicts(resultCode); @@ -888,7 +890,28 @@ END: return errCode; } -int AutoLaunch::GetAutoLaunchProperties(const AutoLaunchParam ¶m, KvDBProperties &properties) +int AutoLaunch::GetAutoLaunchProperties(const AutoLaunchParam ¶m, const DBType &openType, + std::shared_ptr &propertiesPtr) +{ + switch (openType) { + case DBType::DB_KV: { + propertiesPtr = std::make_shared(); + std::shared_ptr kvPtr = std::static_pointer_cast(propertiesPtr); + return GetAutoLaunchKVProperties(param, kvPtr); + } + case DBType::DB_RELATION: { + propertiesPtr = std::make_shared(); + std::shared_ptr rdbPtr = + std::static_pointer_cast(propertiesPtr); + return GetAutoLaunchRelationProperties(param, rdbPtr); + } + default: + return -E_INVALID_ARGS; + } +} + +int AutoLaunch::GetAutoLaunchKVProperties(const AutoLaunchParam ¶m, + const std::shared_ptr &propertiesPtr) { SchemaObject schemaObject; std::string canonicalDir; @@ -898,25 +921,185 @@ int AutoLaunch::GetAutoLaunchProperties(const AutoLaunchParam ¶m, KvDBProper } if (param.option.isEncryptedDb) { - properties.SetPassword(param.option.cipher, param.option.passwd); - } - properties.SetStringProp(KvDBProperties::DATA_DIR, canonicalDir); - properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, param.option.createIfNecessary); - properties.SetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, param.option.createDirByStoreIdOnly); - properties.SetBoolProp(KvDBProperties::MEMORY_MODE, false); - properties.SetBoolProp(KvDBProperties::ENCRYPTED_MODE, param.option.isEncryptedDb); - properties.SetIntProp(KvDBProperties::DATABASE_TYPE, KvDBProperties::SINGLE_VER_TYPE); - properties.SetSchema(schemaObject); + propertiesPtr->SetPassword(param.option.cipher, param.option.passwd); + } + propertiesPtr->SetStringProp(KvDBProperties::DATA_DIR, canonicalDir); + propertiesPtr->SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, param.option.createIfNecessary); + propertiesPtr->SetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, param.option.createDirByStoreIdOnly); + propertiesPtr->SetBoolProp(KvDBProperties::MEMORY_MODE, false); + propertiesPtr->SetBoolProp(KvDBProperties::ENCRYPTED_MODE, param.option.isEncryptedDb); + propertiesPtr->SetIntProp(KvDBProperties::DATABASE_TYPE, KvDBProperties::SINGLE_VER_TYPE); + propertiesPtr->SetSchema(schemaObject); if (RuntimeContext::GetInstance()->IsProcessSystemApiAdapterValid()) { - properties.SetIntProp(KvDBProperties::SECURITY_LABEL, param.option.secOption.securityLabel); - properties.SetIntProp(KvDBProperties::SECURITY_FLAG, param.option.secOption.securityFlag); + propertiesPtr->SetIntProp(KvDBProperties::SECURITY_LABEL, param.option.secOption.securityLabel); + propertiesPtr->SetIntProp(KvDBProperties::SECURITY_FLAG, param.option.secOption.securityFlag); } - properties.SetBoolProp(KvDBProperties::COMPRESS_ON_SYNC, param.option.isNeedCompressOnSync); + propertiesPtr->SetBoolProp(KvDBProperties::COMPRESS_ON_SYNC, param.option.isNeedCompressOnSync); if (param.option.isNeedCompressOnSync) { - properties.SetIntProp(KvDBProperties::COMPRESSION_RATE, + propertiesPtr->SetIntProp(KvDBProperties::COMPRESSION_RATE, ParamCheckUtils::GetValidCompressionRate(param.option.compressionRate)); } - DBCommon::SetDatabaseIds(properties, param.appId, param.userId, param.storeId); + DBCommon::SetDatabaseIds(*propertiesPtr, param.appId, param.userId, param.storeId); return E_OK; } + +int AutoLaunch::GetAutoLaunchRelationProperties(const AutoLaunchParam ¶m, + const std::shared_ptr &propertiesPtr) +{ + if (!ParamCheckUtils::CheckStoreParameter(param.storeId, param.appId, param.userId)) { + LOGE("[AutoLaunch] CheckStoreParameter is invalid."); + return -E_INVALID_ARGS; + } + propertiesPtr->SetStringProp(RelationalDBProperties::DATA_DIR, param.path); + propertiesPtr->SetIdentifier(param.userId, param.appId, param.storeId); + return E_OK; +} + +int AutoLaunch::ExtAutoLaunchRequestCallBack(const std::string &identifier, AutoLaunchParam ¶m, DBType &openType) +{ + std::lock_guard lock(extLock_); + if (autoLaunchRequestCallbackMap_.empty()) { + LOGI("[AutoLaunch] autoLaunchRequestCallbackMap_ is empty"); + return -E_NOT_FOUND; // not E_OK is ok for communicator + } + + bool needOpen = false; + for (const auto &[type, callBack] : autoLaunchRequestCallbackMap_) { + needOpen = callBack(identifier, param); + if (needOpen) { + openType = type; + break; + } + } + + if (!needOpen) { + LOGI("[AutoLaunch] autoLaunchRequestCallback is not need open"); + return -E_NOT_FOUND; // not E_OK is ok for communicator + } + // inner error happened + if (openType >= DBType::DB_INVALID) { + LOGW("[AutoLaunch] Unknown DB Type, Ignore the open request"); + return -E_NOT_FOUND; // not E_OK is ok for communicator + } + return E_OK; +} + +int AutoLaunch::OpenOneKvConnection(AutoLaunchItem &autoLaunchItem) +{ + std::shared_ptr properties = + std::static_pointer_cast(autoLaunchItem.propertiesPtr); + int errCode = E_OK; + IKvDBConnection *conn = KvDBManager::GetDatabaseConnection(*properties, errCode, false); + if (errCode == -E_ALREADY_OPENED) { + LOGI("[AutoLaunch] GetOneConnection user already getkvstore by self"); + } else if (conn == nullptr) { + LOGE("[AutoLaunch] GetOneConnection GetDatabaseConnection failed errCode:%d", errCode); + } + autoLaunchItem.conn = conn; + return errCode; +} + +int AutoLaunch::OpenOneRelationalConnection(AutoLaunchItem &autoLaunchItem) +{ + std::shared_ptr properties = + std::static_pointer_cast(autoLaunchItem.propertiesPtr); + int errCode = E_OK; + auto conn = RelationalStoreInstance::GetDatabaseConnection(*properties, errCode); + if (errCode == -E_ALREADY_OPENED) { + LOGI("[AutoLaunch] GetOneConnection user already openstore by self"); + } else if (conn == nullptr) { + LOGE("[AutoLaunch] GetOneConnection GetDatabaseConnection failed errCode:%d", errCode); + } + autoLaunchItem.conn = conn; + return errCode; +} + +int AutoLaunch::RegisterLifeCycleCallback(AutoLaunchItem &autoLaunchItem, const std::string &identifier, + bool isExt) +{ + int errCode = E_OK; + DatabaseLifeCycleNotifier notifier; + if (isExt) { + notifier = std::bind( + &AutoLaunch::ExtConnectionLifeCycleCallback, this, std::placeholders::_1); + } else { + notifier = std::bind(&AutoLaunch::ConnectionLifeCycleCallback, + this, std::placeholders::_1); + } + switch (autoLaunchItem.type) { + case DBType::DB_KV: + errCode = static_cast(autoLaunchItem.conn)->RegisterLifeCycleCallback(notifier); + break; + case DBType::DB_RELATION: + errCode = + static_cast(autoLaunchItem.conn)->RegisterLifeCycleCallback(notifier); + break; + default: + LOGD("[AutoLaunch] Unknown Type[%d]", autoLaunchItem.type); + break; + } + return errCode; +} + +int AutoLaunch::PragmaAutoSync(AutoLaunchItem &autoLaunchItem) +{ + int errCode = E_OK; + if (autoLaunchItem.type != DBType::DB_KV) { + LOGD("[AutoLaunch] Current Type[%d] Not Support AutoSync Now", autoLaunchItem.type); + return errCode; + } + + bool enAutoSync = autoLaunchItem.isAutoSync; + errCode = static_cast(autoLaunchItem.conn)->Pragma(PRAGMA_AUTO_SYNC, + static_cast(&enAutoSync)); + if (errCode != E_OK) { + LOGE("[AutoLaunch] PRAGMA_AUTO_SYNC failed, errCode:%d", errCode); + return errCode; + } + LOGI("[AutoLaunch] set PRAGMA_AUTO_SYNC ok, enAutoSync=%d", enAutoSync); + return errCode; +} + +void AutoLaunch::TryCloseKvConnection(AutoLaunchItem &autoLaunchItem) +{ + LOGI("[AutoLaunch] TryCloseKvConnection"); + if (autoLaunchItem.conn == nullptr) { + LOGI("[AutoLaunch] TryCloseConnection conn is nullptr, do nothing"); + return; + } + IKvDBConnection *kvConn = static_cast(autoLaunchItem.conn); + int errCode = kvConn->RegisterLifeCycleCallback(nullptr); + if (errCode != E_OK) { + LOGE("[AutoLaunch] TryCloseConnection RegisterLifeCycleCallback failed errCode:%d", errCode); + } + if (autoLaunchItem.observerHandle != nullptr) { + errCode = kvConn->UnRegisterObserver(autoLaunchItem.observerHandle); + if (errCode != E_OK) { + LOGE("[AutoLaunch] TryCloseConnection UnRegisterObserver failed errCode:%d", errCode); + } + autoLaunchItem.observerHandle = nullptr; + } + errCode = KvDBManager::ReleaseDatabaseConnection(kvConn); + if (errCode != E_OK) { + LOGE("[AutoLaunch] TryCloseConnection ReleaseDatabaseConnection failed errCode:%d", errCode); + } +} + +void AutoLaunch::TryCloseRelationConnection(AutoLaunchItem &autoLaunchItem) +{ + LOGI("[AutoLaunch] TryCloseRelationConnection"); + if (autoLaunchItem.conn == nullptr) { + LOGI("[AutoLaunch] TryCloseConnection conn is nullptr, do nothing"); + return; + } + RelationalStoreConnection *rdbConn = static_cast(autoLaunchItem.conn); + int errCode = rdbConn->RegisterLifeCycleCallback(nullptr); + if (errCode != E_OK) { + LOGE("[AutoLaunch] TryCloseConnection RegisterLifeCycleCallback failed errCode:%d", errCode); + } + errCode = rdbConn->Close(); + if (errCode != E_OK) { + LOGE("[AutoLaunch] TryCloseConnection close connection failed errCode:%d", errCode); + } +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp index f3d08cc60..10ab3e063 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp @@ -398,9 +398,9 @@ void RuntimeContextImpl::GetAutoLaunchSyncDevices(const std::string &identifier, return autoLaunch_.GetAutoLaunchSyncDevices(identifier, devices); } -void RuntimeContextImpl::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) +void RuntimeContextImpl::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback, DBType type) { - autoLaunch_.SetAutoLaunchRequestCallback(callback); + autoLaunch_.SetAutoLaunchRequestCallback(callback, type); } NotificationChain::Listener *RuntimeContextImpl::RegisterLockStatusLister(const LockStatusNotifier &action, diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h index 72ff30636..ac645528d 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h @@ -76,7 +76,7 @@ public: void GetAutoLaunchSyncDevices(const std::string &identifier, std::vector &devices) const override; - void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) override; + void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback, DBType type) override; NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errCode) override; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h index ed5add667..f1ca114b2 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h @@ -79,7 +79,10 @@ public: int GetInt64(int64_t &outVal) const; int GetDouble(double &outVal) const; int GetBlob(Blob *&outVal) const; + int SetBlob(const Blob &val); int GetBlob(Blob &outVal) const; + int SetText(const std::string &val); + int SetText(const uint8_t *val, uint32_t length); int GetText(std::string &outVal) const; void ResetValue(); int GetBlobLength(uint32_t &length) const; @@ -104,6 +107,7 @@ public: int GetString(const std::string &fieldName, std::string &outValue) const; int GetBlob(const std::string &fieldName, Blob &blob) const; void PutDataValue(const std::string &fieldName, const DataValue &value); + int GetDataValue(const std::string &fieldName, DataValue &value) const; private: mutable std::map fieldData; }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp index f144d07ab..53b8aae72 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp @@ -526,14 +526,15 @@ DBStatus KvStoreDelegateManager::EnableKvStoreAutoLaunch(const std::string &user return DB_ERROR; } AutoLaunchParam param{ userId, appId, storeId, option, notifier }; - KvDBProperties properties; - int errCode = AutoLaunch::GetAutoLaunchProperties(param, properties); + std::shared_ptr ptr = std::make_shared(); + int errCode = AutoLaunch::GetAutoLaunchProperties(param, DBType::DB_KV, ptr); if (errCode != E_OK) { LOGE("[KvStoreManager] Enable auto launch failed:%d", errCode); return TransferDBErrno(errCode); } - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(properties, notifier, option); + std::shared_ptr kvPtr = std::static_pointer_cast(ptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(*kvPtr, notifier, option); if (errCode != E_OK) { LOGE("[KvStoreManager] Enable auto launch failed:%d", errCode); return TransferDBErrno(errCode); @@ -562,7 +563,7 @@ DBStatus KvStoreDelegateManager::DisableKvStoreAutoLaunch(const std::string &use void KvStoreDelegateManager::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) { - RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(callback); + RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(callback, DBType::DB_KV); } std::string KvStoreDelegateManager::GetKvStoreIdentifier(const std::string &userId, const std::string &appId, diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index 96e70dd48..06718e615 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -119,17 +119,17 @@ DataValue &DataValue::operator=(const DataValue &dataValue) ResetValue(); switch (dataValue.type_) { case StorageType::STORAGE_TYPE_BOOL: - dataValue.GetBool(this->value_.bValue); + (void)dataValue.GetBool(this->value_.bValue); break; case StorageType::STORAGE_TYPE_INTEGER: - dataValue.GetInt64(this->value_.iValue); + (void)dataValue.GetInt64(this->value_.iValue); break; case StorageType::STORAGE_TYPE_REAL: - dataValue.GetDouble(this->value_.dValue); + (void)dataValue.GetDouble(this->value_.dValue); break; case StorageType::STORAGE_TYPE_BLOB: case StorageType::STORAGE_TYPE_TEXT: - dataValue.GetBlob(this->value_.blobPtr); + (void)dataValue.GetBlob(this->value_.blobPtr); break; default: break; @@ -183,43 +183,13 @@ DataValue &DataValue::operator=(const double &doubleVal) DataValue &DataValue::operator=(const Blob &blob) { - ResetValue(); - if (blob.GetSize() <= 0) { - return *this; - } - value_.blobPtr = new (std::nothrow) Blob(); - if (value_.blobPtr == nullptr) { - return *this; - } - type_ = StorageType::STORAGE_TYPE_BLOB; - if (blob.GetSize() > 0) { - value_.blobPtr->WriteBlob(blob.GetData(), blob.GetSize()); - } + (void)SetBlob(blob); return *this; } -int DataValue::Set(Blob *&blob) -{ - ResetValue(); - if (blob == nullptr || blob->GetSize() <= 0) { - LOGE("Transfer Blob to DataValue failed."); - return -E_INVALID_ARGS; - } - type_ = StorageType::STORAGE_TYPE_BLOB; - value_.blobPtr = blob; - blob = nullptr; - return E_OK; -} - DataValue &DataValue::operator=(const std::string &string) { - ResetValue(); - value_.blobPtr = new (std::nothrow) Blob(); - if (value_.blobPtr == nullptr) { - return *this; - } - type_ = StorageType::STORAGE_TYPE_TEXT; - value_.blobPtr->WriteBlob(reinterpret_cast(string.c_str()), string.size()); + (void)SetText(string); return *this; } @@ -297,6 +267,24 @@ int DataValue::GetBlob(Blob *&outVal) const return outVal->WriteBlob(value_.blobPtr->GetData(), value_.blobPtr->GetSize()); } +int DataValue::SetBlob(const Blob &val) +{ + ResetValue(); + if (val.GetSize() <= 0) { + return E_OK; + } + value_.blobPtr = new(std::nothrow) Blob(); + if (value_.blobPtr == nullptr) { + return -E_OUT_OF_MEMORY; + } + type_ = StorageType::STORAGE_TYPE_BLOB; + int errCode = E_OK; + if (val.GetSize() > 0) { + errCode = value_.blobPtr->WriteBlob(val.GetData(), val.GetSize()); + } + return errCode; +} + int DataValue::GetBlob(Blob &outVal) const { if (type_ != StorageType::STORAGE_TYPE_BLOB && type_ != StorageType::STORAGE_TYPE_TEXT) { @@ -305,6 +293,22 @@ int DataValue::GetBlob(Blob &outVal) const return outVal.WriteBlob(value_.blobPtr->GetData(), value_.blobPtr->GetSize()); } +int DataValue::SetText(const std::string &val) +{ + return SetText(reinterpret_cast(val.c_str()), val.length()); +} + +int DataValue::SetText(const uint8_t *val, uint32_t length) +{ + ResetValue(); + value_.blobPtr = new(std::nothrow) Blob(); + if (value_.blobPtr == nullptr) { + return -E_OUT_OF_MEMORY; + } + type_ = StorageType::STORAGE_TYPE_TEXT; + return value_.blobPtr->WriteBlob(val, length); +} + int DataValue::GetText(std::string &outValue) const { if (type_ != StorageType::STORAGE_TYPE_TEXT) { @@ -312,6 +316,10 @@ int DataValue::GetText(std::string &outValue) const } const uint8_t *data = value_.blobPtr->GetData(); uint32_t len = value_.blobPtr->GetSize(); + if (len == 0) { + outValue = ""; + return E_OK; + } outValue.resize(len); outValue.assign(data, data + len); return E_OK; @@ -436,5 +444,14 @@ void ObjectData::PutDataValue(const std::string &fieldName, const DataValue &val { fieldData[fieldName] = value; } + +int ObjectData::GetDataValue(const std::string &fieldName, DataValue &value) const +{ + if (fieldData.find(fieldName) == fieldData.end()) { + return -E_NOT_FOUND; + } + value = fieldData[fieldName]; + return E_OK; +} } // namespace DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 6ed6056a4..4d85bf7da 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -124,6 +124,7 @@ std::string RelationalStoreManager::GetDistributedTableName(const std::string &d void RelationalStoreManager::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) { + RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(callback, DBType::DB_RELATION); } std::string RelationalStoreManager::GetRelationalStoreIdentifier(const std::string &userId, const std::string &appId, diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index 9c110df5c..378b72c98 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -108,6 +108,7 @@ public: int RegisterSchemaChangedCallback(const std::function &callback) override; void NotifySchemaChanged(); + int GetCompressionAlgo(std::set &algorithmSet) const override; private: SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode, OperatePerm perm) const; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/sync_generic_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/sync_generic_interface.h index 606808111..600e1c3a8 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/sync_generic_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/sync_generic_interface.h @@ -129,6 +129,16 @@ public: { return -E_NOT_SUPPORT; } + + virtual int GetCompressionAlgo(std::set &algorithmSet) const + { + return -E_NOT_SUPPORT; + } + + virtual bool CheckCompatible(const std::string &schema) const + { + return false; + } }; } #endif // SYNC_GENERIC_INTERFACE_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index b34d6459c..1f230f737 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -14,6 +14,8 @@ */ #ifdef RELATIONAL_STORE #include "relational_sync_able_storage.h" + +#include "data_compression.h" #include "platform_specific.h" #include "generic_single_ver_kv_entry.h" @@ -481,5 +483,11 @@ void RelationalSyncAbleStorage::NotifySchemaChanged() onSchemaChanged_(); } } +int RelationalSyncAbleStorage::GetCompressionAlgo(std::set &algorithmSet) const +{ + algorithmSet.clear(); + DataCompression::GetCompressionAlgo(algorithmSet); + return E_OK; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp index cbc1fbb56..c74ac8aa8 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp @@ -150,7 +150,7 @@ void SingleVerSyncTaskContext::SetContinueToken(ContinueToken token) void SingleVerSyncTaskContext::ReleaseContinueToken() { if (token_ != nullptr) { - static_cast(syncInterface_)->ReleaseContinueToken(token_); + static_cast(syncInterface_)->ReleaseContinueToken(token_); token_ = nullptr; } } @@ -461,7 +461,7 @@ CompressAlgorithm SingleVerSyncTaskContext::ChooseCompressAlgo() const return CompressAlgorithm::NONE; } std::set localAlgorithmSet; - (void)(static_cast(syncInterface_))->GetCompressionAlgo(localAlgorithmSet); + (void)(static_cast(syncInterface_))->GetCompressionAlgo(localAlgorithmSet); std::set algoIntersection; set_intersection(remoteAlgo.begin(), remoteAlgo.end(), localAlgorithmSet.begin(), localAlgorithmSet.end(), inserter(algoIntersection, algoIntersection.begin())); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp index 7ad339b78..b4a84e5d3 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp @@ -666,7 +666,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch007, TestSize.Level3) * @tc.expected: step1. success. */ RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback( - std::bind(AutoLaunchCallBack, std::placeholders::_1, std::placeholders::_2, observer, true)); + std::bind(AutoLaunchCallBack, std::placeholders::_1, std::placeholders::_2, observer, true), DBType::DB_KV); /** * @tc.steps: step2. RunCommunicatorLackCallback * @tc.expected: step2. success. @@ -709,7 +709,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch007, TestSize.Level3) g_statusMap.clear(); g_finished = false; } - RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr); + RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr, DBType::DB_KV); delete observer; } @@ -729,7 +729,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch008, TestSize.Level3) * @tc.expected: step1. success. */ RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback( - std::bind(AutoLaunchCallBack, std::placeholders::_1, std::placeholders::_2, observer, false)); + std::bind(AutoLaunchCallBack, std::placeholders::_1, std::placeholders::_2, observer, false), DBType::DB_KV); /** * @tc.steps: step2. RunCommunicatorLackCallback * @tc.expected: step2. success. @@ -747,7 +747,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch008, TestSize.Level3) std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_TIME)); EXPECT_TRUE(observer->GetCallCount() == 0); EXPECT_TRUE(g_finished == false); - RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr); + RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr, DBType::DB_KV); delete observer; } @@ -766,7 +766,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch009, TestSize.Level3) * @tc.steps: step1. SetAutoLaunchRequestCallback * @tc.expected: step1. success. */ - RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(AutoLaunchCallBackBadParam); + RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(AutoLaunchCallBackBadParam, DBType::DB_KV); /** * @tc.steps: step2. RunCommunicatorLackCallback * @tc.expected: step2. success. @@ -789,7 +789,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch009, TestSize.Level3) g_statusMap.clear(); g_finished = false; } - RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr); + RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr, DBType::DB_KV); delete observer; } @@ -809,9 +809,9 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch010, TestSize.Level3) * @tc.expected: step1. success. */ RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback( - std::bind(AutoLaunchCallBack, std::placeholders::_1, std::placeholders::_2, observer, false)); + std::bind(AutoLaunchCallBack, std::placeholders::_1, std::placeholders::_2, observer, false), DBType::DB_KV); - RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr); + RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr, DBType::DB_KV); /** * @tc.steps: step2. RunCommunicatorLackCallback * @tc.expected: step2. success. @@ -828,7 +828,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch010, TestSize.Level3) std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_TIME)); EXPECT_TRUE(observer->GetCallCount() == 0); EXPECT_TRUE(g_finished == false); - RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr); + RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr, DBType::DB_KV); delete observer; } @@ -941,7 +941,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch013, TestSize.Level3) KvStoreObserverUnitTest *observer = new (std::nothrow) KvStoreObserverUnitTest; ASSERT_TRUE(observer != nullptr); RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback( - std::bind(AutoLaunchCallBack, std::placeholders::_1, std::placeholders::_2, observer, true)); + std::bind(AutoLaunchCallBack, std::placeholders::_1, std::placeholders::_2, observer, true), DBType::DB_KV); /** * @tc.steps: step2. RunOnConnectCallback RunCommunicatorLackCallback @@ -988,7 +988,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch013, TestSize.Level3) g_statusMap.clear(); g_finished = false; } - RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr); + RuntimeContext::GetInstance()->SetAutoLaunchRequestCallback(nullptr, DBType::DB_KV); delete observer; /** * @tc.steps: step4. param A B disable diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index 8ec7605e7..06eec91d2 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -35,15 +35,25 @@ using namespace DistributedDBUnitTest; namespace { const std::string DEVICE_B = "deviceB"; const std::string g_tableName = "TEST_TABLE"; + const std::string g_syncTableName = "naturalbase_rdb_aux_" + + g_tableName + "_" + DBCommon::TransferStringToHex(DBCommon::TransferHashString(DEVICE_B)); RelationalStoreManager g_mgr(APP_ID, USER_ID); std::string g_testDir; std::string g_dbDir; + std::string g_id; DistributedDBToolsUnitTest g_tool; - DBStatus g_kvDelegateStatus = INVALID_ARGS; RelationalStoreDelegate* g_kvDelegatePtr = nullptr; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; RelationalVirtualDevice *g_deviceB = nullptr; + std::vector g_fieldInfoList; + + void OpenStore() + { + RelationalStoreDelegate::Option option; + g_mgr.OpenStore(g_dbDir, STORE_ID_1, option, g_kvDelegatePtr); + ASSERT_TRUE(g_kvDelegatePtr != nullptr); + } int GetDB(sqlite3 *&db) { @@ -116,7 +126,7 @@ namespace { dataMap["ID"] = id; rowData.push_back(dataMap["ID"]); std::string val = "test"; - dataMap["NAME"] = val; + dataMap["NAME"].SetText(val); rowData.push_back(dataMap["NAME"]); dataMap["AGE"] = INT64_MAX; rowData.push_back(dataMap["AGE"]); @@ -124,7 +134,7 @@ namespace { void InsertFieldInfo() { - std::vector localFieldInfo; + g_fieldInfoList.clear(); FieldInfo columnFirst; columnFirst.SetFieldName("ID"); columnFirst.SetStorageType(StorageType::STORAGE_TYPE_INTEGER); @@ -134,10 +144,10 @@ namespace { FieldInfo columnThird; columnThird.SetFieldName("AGE"); columnThird.SetStorageType(StorageType::STORAGE_TYPE_INTEGER); - localFieldInfo.push_back(columnFirst); - localFieldInfo.push_back(columnSecond); - localFieldInfo.push_back(columnThird); - g_deviceB->SetLocalFieldInfo(localFieldInfo); + g_fieldInfoList.push_back(columnFirst); + g_fieldInfoList.push_back(columnSecond); + g_fieldInfoList.push_back(columnThird); + g_deviceB->SetLocalFieldInfo(g_fieldInfoList); } void BlockSync(SyncMode syncMode, DBStatus exceptStatus) @@ -158,6 +168,69 @@ namespace { } } + int PrepareSelect(sqlite3 *db, sqlite3_stmt *&statement, const std::string &table) + { + const std::string sql = "SELECT * FROM " + table; + return sqlite3_prepare_v2(db, sql.c_str(), -1, &statement, nullptr); + } + + void GetDataValue(sqlite3_stmt *statement, int row, int col, DataValue &dataValue) + { + int type = sqlite3_column_type(statement, col); + switch (type) { + case SQLITE_INTEGER: { + dataValue = static_cast(sqlite3_column_int64(statement, col)); + break; + } + case SQLITE_FLOAT: { + dataValue = sqlite3_column_double(statement, col); + break; + } + case SQLITE_TEXT: { + dataValue.SetText(std::string(reinterpret_cast(sqlite3_column_text(statement, col)))); + break; + } + case SQLITE_BLOB: { + std::vector blobValue; + (void)SQLiteUtils::GetColumnBlobValue(statement, col, blobValue); + Blob blob; + blob.WriteBlob(blobValue.data(), static_cast(blobValue.size())); + dataValue.SetBlob(blob); + break; + } + case SQLITE_NULL: + break; + default: + LOGW("unknown type[%d] row[%llu] column[%d] ignore", type, row, col); + } + } + + void GetSyncDataStep(std::vector &dataList, sqlite3_stmt *statement) + { + int columnCount = sqlite3_column_count(statement); + RowData rowData; + for (int col = 0; col < columnCount; ++col) { + DataValue dataValue; + GetDataValue(statement, static_cast(dataList.size() + 1), col, dataValue); + rowData.push_back(std::move(dataValue)); + } + dataList.push_back(rowData); + } + + void GetSyncData(sqlite3 *db, std::vector &dataList) + { + sqlite3_stmt *statement = nullptr; + EXPECT_EQ(PrepareSelect(db, statement, g_syncTableName), SQLITE_OK); + while (true) { + int rc = sqlite3_step(statement); + if (rc != SQLITE_ROW) { + LOGD("GetSyncData Exist by code[%d]", rc); + break; + } + GetSyncDataStep(dataList, statement); + } + } + void InsertValueToDB(RowData &rowData) { sqlite3 *db = nullptr; @@ -166,7 +239,7 @@ namespace { sqlite3_close(db); } - void PrepareEnvironment(RowData &rowData) + void PrepareEnvironment(std::map dataMap) { sqlite3 *db = nullptr; EXPECT_EQ(GetDB(db), SQLITE_OK); @@ -177,27 +250,57 @@ namespace { sqlite3_close(db); InsertFieldInfo(); - std::map dataMap; + RowData rowData; GenerateValue(rowData, dataMap); InsertValueToDB(rowData); + } + + void PrepareVirtualEnvironment(RowData &rowData) + { + sqlite3 *db = nullptr; + EXPECT_EQ(GetDB(db), SQLITE_OK); + EXPECT_EQ(CreateTable(db), SQLITE_OK); + EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName), OK); + + sqlite3_close(db); - rowData.clear(); + std::map dataMap; + GenerateValue(rowData, dataMap); + VirtualRowData virtualRowData; for (const auto &item : dataMap) { - rowData.push_back(item.second); + virtualRowData.objectData.PutDataValue(item.first, item.second); } + virtualRowData.logInfo.timestamp = 1; + g_deviceB->PutData(g_tableName, {virtualRowData}); + InsertFieldInfo(); } - void CheckVirtualData(RowData &rowData) + void CheckData(RowData &rowData) { - std::vector targetData; + std::vector dataList; + sqlite3 *db = nullptr; + EXPECT_EQ(GetDB(db), SQLITE_OK); + GetSyncData(db, dataList); + sqlite3_close(db); + + ASSERT_EQ(dataList.size(), static_cast(1)); + for (size_t j = 0; j < dataList[0].size(); ++j) { + EXPECT_TRUE(dataList[0][j] == rowData[j]); + } + } + + void CheckVirtualData(std::map data) + { + std::vector targetData; g_deviceB->GetAllSyncData(g_tableName, targetData); - for (auto &item : targetData) { - for (int j = 0; j < static_cast(item.rowData.size()); ++j) { - LOGD("index %d actual_val[%s] except_val[%s]", - j, item.rowData[j].ToString().c_str(), rowData[j].ToString().c_str()); - EXPECT_TRUE(item.rowData[j] == rowData[j]); - } + ASSERT_EQ(targetData.size(), 1u); + for (auto &[field, value] : data) { + DataValue target; + EXPECT_EQ(targetData[0].objectData.GetDataValue(field, target), E_OK); + LOGD("field %s actual_val[%s] except_val[%s]", + field.c_str(), target.ToString().c_str(), value.ToString().c_str()); + EXPECT_TRUE(target == value); } } } @@ -217,13 +320,12 @@ void DistributedDBRelationalVerP2PSyncTest::SetUpTestCase() */ DistributedDBToolsUnitTest::TestDirInit(g_testDir); g_dbDir = g_testDir + "/test.db"; - sqlite3 *db = nullptr; - ASSERT_EQ(GetDB(db), SQLITE_OK); - sqlite3_close(db); g_communicatorAggregator = new (std::nothrow) VirtualCommunicatorAggregator(); ASSERT_TRUE(g_communicatorAggregator != nullptr); RuntimeContext::GetInstance()->SetCommunicatorAggregator(g_communicatorAggregator); + + g_id = g_mgr.GetRelationalStoreIdentifier(APP_ID, USER_ID, STORE_ID_1); } void DistributedDBRelationalVerP2PSyncTest::TearDownTestCase() @@ -244,9 +346,10 @@ void DistributedDBRelationalVerP2PSyncTest::SetUp(void) /** * @tc.setup: create virtual device B, and get a KvStoreNbDelegate as deviceA */ - g_kvDelegateStatus = g_mgr.OpenStore(g_dbDir, "Relational_default_id", {}, g_kvDelegatePtr); - ASSERT_TRUE(g_kvDelegateStatus == OK); - ASSERT_TRUE(g_kvDelegatePtr != nullptr); + sqlite3 *db = nullptr; + ASSERT_EQ(GetDB(db), SQLITE_OK); + sqlite3_close(db); + OpenStore(); g_deviceB = new (std::nothrow) RelationalVirtualDevice(DEVICE_B); ASSERT_TRUE(g_deviceB != nullptr); auto *syncInterfaceB = new (std::nothrow) VirtualRelationalVerSyncDBInterface(); @@ -283,86 +386,80 @@ void DistributedDBRelationalVerP2PSyncTest::TearDown(void) * @tc.name: Normal Sync 001 * @tc.desc: Test normal push sync for add data. * @tc.type: FUNC -* @tc.require: +* @tc.require: AR000GK58N * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync001, TestSize.Level1) { - RowData rowData; - PrepareEnvironment(rowData); + std::map dataMap; + PrepareEnvironment(dataMap); BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); - CheckVirtualData(rowData); + CheckVirtualData(dataMap); } /** * @tc.name: Normal Sync 002 * @tc.desc: Test normal pull sync for add data. * @tc.type: FUNC -* @tc.require: +* @tc.require: AR000GK58N * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync002, TestSize.Level1) { - RowData rowData; - PrepareEnvironment(rowData); + std::map dataMap; + PrepareEnvironment(dataMap); - std::vector devices = {DEVICE_B}; Query query = Query::Select(g_tableName); g_deviceB->GenericVirtualDevice::Sync(DistributedDB::SYNC_MODE_PULL_ONLY, query, true); - CheckVirtualData(rowData); + CheckVirtualData(dataMap); } /** * @tc.name: Normal Sync 003 * @tc.desc: Test normal push sync for update data. * @tc.type: FUNC -* @tc.require: +* @tc.require: AR000GK58N * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync003, TestSize.Level1) { - RowData rowData; - PrepareEnvironment(rowData); + std::map dataMap; + PrepareEnvironment(dataMap); BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); - CheckVirtualData(rowData); + CheckVirtualData(dataMap); int64_t val = 0; - std::map dataMap; - rowData.clear(); + RowData rowData; GenerateValue(rowData, dataMap); EXPECT_EQ(rowData[rowData.size() - 1].GetInt64(val), E_OK); rowData[rowData.size() - 1] = static_cast(1); InsertValueToDB(rowData); BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); - rowData.clear(); dataMap["AGE"] = static_cast(1); - for (const auto &item : dataMap) { - rowData.push_back(item.second); - } - CheckVirtualData(rowData); + CheckVirtualData(dataMap); } /** * @tc.name: Normal Sync 004 * @tc.desc: Test normal push sync for delete data. * @tc.type: FUNC -* @tc.require: +* @tc.require: AR000GK58N * @tc.author: zhangqiquan */ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync004, TestSize.Level1) { - RowData rowData; - PrepareEnvironment(rowData); + std::map dataMap; + PrepareEnvironment(dataMap); BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); - CheckVirtualData(rowData); + CheckVirtualData(dataMap); sqlite3 *db = nullptr; EXPECT_EQ(GetDB(db), SQLITE_OK); @@ -372,11 +469,194 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync004, TestSize.Level1) BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); - std::vector dataList; + std::vector dataList; EXPECT_EQ(g_deviceB->GetAllSyncData(g_tableName, dataList), E_OK); EXPECT_EQ(static_cast(dataList.size()), 1); for (const auto &item : dataList) { EXPECT_EQ(item.logInfo.flag, DataItem::DELETE_FLAG); } } + +/** +* @tc.name: Normal Sync 005 +* @tc.desc: Test normal push sync for add data. +* @tc.type: FUNC +* @tc.require: AR000GK58N +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync005, TestSize.Level1) +{ + RowData rowData; + PrepareVirtualEnvironment(rowData); + + Query query = Query::Select(g_tableName); + g_deviceB->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, true); + + CheckData(rowData); +} + +/** +* @tc.name: Normal Sync 006 +* @tc.desc: Test normal pull sync for add data. +* @tc.type: FUNC +* @tc.require: AR000GK58N +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync006, TestSize.Level1) +{ + RowData rowData; + PrepareVirtualEnvironment(rowData); + + BlockSync(SYNC_MODE_PULL_ONLY, OK); + + CheckData(rowData); +} + +/** +* @tc.name: AutoLaunchSync 001 +* @tc.desc: Test rdb autoLaunch success when callback return true. +* @tc.type: FUNC +* @tc.require: AR000GK58N +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync001, TestSize.Level3) +{ + /** + * @tc.steps: step1. open rdb store, create distribute table and insert data + */ + RowData rowData; + PrepareVirtualEnvironment(rowData); + + /** + * @tc.steps: step2. set auto launch callBack + */ + const AutoLaunchRequestCallback callback = [](const std::string &identifier, AutoLaunchParam ¶m){ + if (g_id != identifier) { + return false; + } + param.path = g_dbDir; + param.appId = APP_ID; + param.userId = USER_ID; + param.storeId = STORE_ID_1; + return true; + }; + g_mgr.SetAutoLaunchRequestCallback(callback); + /** + * @tc.steps: step3. close store ensure communicator has closed + */ + g_mgr.CloseStore(g_kvDelegatePtr); + g_kvDelegatePtr = nullptr; + /** + * @tc.steps: step4. RunCommunicatorLackCallback to autolaunch store + */ + LabelType labelType(g_id.begin(), g_id.end()); + g_communicatorAggregator->RunCommunicatorLackCallback(labelType); + std::this_thread::sleep_for(std::chrono::seconds(1)); + /** + * @tc.steps: step5. Call sync expect sync successful + */ + Query query = Query::Select(g_tableName); + EXPECT_EQ(g_deviceB->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, true), E_OK); + /** + * @tc.steps: step6. check sync data ensure sync successful + */ + CheckData(rowData); + + RelationalStoreDelegate::Option option; + g_mgr.OpenStore(g_dbDir, STORE_ID_1, option, g_kvDelegatePtr); + ASSERT_TRUE(g_kvDelegatePtr != nullptr); + + OpenStore(); + std::this_thread::sleep_for(std::chrono::minutes(1)); +} + +/** +* @tc.name: AutoLaunchSync 002 +* @tc.desc: Test rdb autoLaunch failed when callback return false. +* @tc.type: FUNC +* @tc.require: AR000GK58N +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync002, TestSize.Level3) +{ + /** + * @tc.steps: step1. open rdb store, create distribute table and insert data + */ + RowData rowData; + PrepareVirtualEnvironment(rowData); + + /** + * @tc.steps: step2. set auto launch callBack + */ + const AutoLaunchRequestCallback callback = [](const std::string &identifier, AutoLaunchParam ¶m){ + return false; + }; + g_mgr.SetAutoLaunchRequestCallback(callback); + /** + * @tc.steps: step2. close store ensure communicator has closed + */ + g_mgr.CloseStore(g_kvDelegatePtr); + g_kvDelegatePtr = nullptr; + /** + * @tc.steps: step3. store cann't autoLaunch because callback return false + */ + LabelType labelType(g_id.begin(), g_id.end()); + g_communicatorAggregator->RunCommunicatorLackCallback(labelType); + std::this_thread::sleep_for(std::chrono::seconds(1)); + /** + * @tc.steps: step4. Call sync expect sync fail + */ + Query query = Query::Select(g_tableName); + SyncOperation::UserCallback callBack = [](const std::map &statusMap) { + for (const auto &entry : statusMap) { + EXPECT_EQ(entry.second, static_cast(SyncOperation::OP_COMM_ABNORMAL)); + } + }; + EXPECT_EQ(g_deviceB->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, callBack, true), E_OK); + + OpenStore(); + std::this_thread::sleep_for(std::chrono::minutes(1)); +} + +/** +* @tc.name: AutoLaunchSync 003 +* @tc.desc: Test rdb autoLaunch failed when callback is nullptr. +* @tc.type: FUNC +* @tc.require: AR000GK58N +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync003, TestSize.Level3) +{ + /** + * @tc.steps: step1. open rdb store, create distribute table and insert data + */ + RowData rowData; + PrepareVirtualEnvironment(rowData); + + g_mgr.SetAutoLaunchRequestCallback(nullptr); + /** + * @tc.steps: step2. close store ensure communicator has closed + */ + g_mgr.CloseStore(g_kvDelegatePtr); + g_kvDelegatePtr = nullptr; + /** + * @tc.steps: step3. store cann't autoLaunch because callback is nullptr + */ + LabelType labelType(g_id.begin(), g_id.end()); + g_communicatorAggregator->RunCommunicatorLackCallback(labelType); + std::this_thread::sleep_for(std::chrono::seconds(1)); + /** + * @tc.steps: step4. Call sync expect sync fail + */ + Query query = Query::Select(g_tableName); + SyncOperation::UserCallback callBack = [](const std::map &statusMap) { + for (const auto &entry : statusMap) { + EXPECT_EQ(entry.second, static_cast(SyncOperation::OP_COMM_ABNORMAL)); + } + }; + EXPECT_EQ(g_deviceB->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, callBack, true), E_OK); + + OpenStore(); + std::this_thread::sleep_for(std::chrono::minutes(1)); +} #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp index f002c34f5..745bb4df3 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp @@ -19,6 +19,7 @@ #include "distributeddb_tools_unit_test.h" #include "isyncer.h" #include "single_ver_sync_state_machine.h" +#include "single_ver_sync_task_context.h" #include "sync_types.h" #include "virtual_single_ver_sync_db_Interface.h" #include "virtual_time_sync_communicator.h" diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp index 012e14141..0154d235e 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp @@ -212,7 +212,13 @@ int GenericVirtualDevice::Sync(SyncMode mode, bool wait) int GenericVirtualDevice::Sync(SyncMode mode, const Query &query, bool wait) { - auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, nullptr, wait); + return Sync(mode, query, nullptr, wait); +} + +int GenericVirtualDevice::Sync(SyncMode mode, const Query &query, + const SyncOperation::UserCallback &callBack, bool wait) +{ + auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, callBack, wait); if (operation == nullptr) { return -E_OUT_OF_MEMORY; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h index 388a85152..96795c027 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h @@ -42,6 +42,7 @@ public: TimeOffset GetLocalTimeOffset() const; virtual int Sync(SyncMode mode, bool wait); virtual int Sync(SyncMode mode, const Query &query, bool wait); + virtual int Sync(SyncMode mode, const Query &query, const SyncOperation::UserCallback &callBack, bool wait); protected: ICommunicator *communicateHandle_; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp index 16272ff4e..d78398d32 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp @@ -24,16 +24,22 @@ RelationalVirtualDevice::~RelationalVirtualDevice() { } -int RelationalVirtualDevice::PutData(const std::string &tableName, const std::vector &dataList) +int RelationalVirtualDevice::PutData(const std::string &tableName, const std::vector &dataList) { return static_cast(storage_)->PutLocalData(dataList, tableName); } -int RelationalVirtualDevice::GetAllSyncData(const std::string &tableName, std::vector &data) +int RelationalVirtualDevice::GetAllSyncData(const std::string &tableName, std::vector &data) { return static_cast(storage_)->GetAllSyncData(tableName, data); } +int RelationalVirtualDevice::GetSyncData(const std::string &tableName, + const std::string &hashKey, VirtualRowData &data) +{ + return static_cast(storage_)->GetSyncData(tableName, hashKey, data); +} + void RelationalVirtualDevice::SetLocalFieldInfo(const std::vector &localFieldInfo) { static_cast(storage_)->SetLocalFieldInfo(localFieldInfo); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h index 32f974bcd..b3232f9c9 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h @@ -16,9 +16,10 @@ #define RELATIONAL_VIRTUAL_DEVICE_H #ifdef RELATIONAL_STORE +#include "data_transformer.h" #include "generic_virtual_device.h" #include "relational_schema_object.h" -#include "data_transformer.h" +#include "virtual_relational_ver_sync_db_interface.h" namespace DistributedDB { class RelationalVirtualDevice final : public GenericVirtualDevice { @@ -26,8 +27,9 @@ public: explicit RelationalVirtualDevice(const std::string &deviceId); ~RelationalVirtualDevice() override; - int PutData(const std::string &tableName, const std::vector &dataList); - int GetAllSyncData(const std::string &tableName, std::vector &data); + int PutData(const std::string &tableName, const std::vector &dataList); + int GetAllSyncData(const std::string &tableName, std::vector &data); + int GetSyncData(const std::string &tableName, const std::string &hashKey, VirtualRowData &data); void SetLocalFieldInfo(const std::vector &localFieldInfo); int Sync(SyncMode mode, bool wait) override; }; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp index 42b703606..3a6678ae8 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp @@ -78,37 +78,30 @@ int VirtualRelationalVerSyncDBInterface::PutSyncDataWithQuery(const QueryObject if (errCode != E_OK) { return errCode; } - std::vector dataList; - std::set hashKeySet; for (const auto &optRowDataWithLog : optTableDataWithLog.dataList) { - RowDataWithLog rowDataWithLog; - rowDataWithLog.logInfo = optRowDataWithLog.logInfo; + VirtualRowData virtualRowData; + virtualRowData.logInfo = optRowDataWithLog.logInfo; + int index = 0; for (const auto &optItem : optRowDataWithLog.optionalData) { - if (!optItem.has_value()) { - continue; + DataValue dataValue; + if (optItem.has_value()) { + dataValue = std::move(optItem.value()); } - rowDataWithLog.rowData.push_back(optItem.value()); LOGD("type:%d", optItem.value().GetType()); + virtualRowData.objectData.PutDataValue(localFieldInfo_[index].GetFieldName(), dataValue); + index++; } - hashKeySet.insert(rowDataWithLog.logInfo.hashKey); - dataList.push_back(rowDataWithLog); + syncData_[object.GetTableName()][virtualRowData.logInfo.hashKey] = virtualRowData; } LOGD("tableName %s", optTableDataWithLog.tableName.c_str()); - for (const auto &item : syncData_[optTableDataWithLog.tableName]) { - if (hashKeySet.find(item.logInfo.hashKey) == hashKeySet.end()) { - dataList.push_back(item); - } - } - syncData_[optTableDataWithLog.tableName] = dataList; - return errCode; } -int VirtualRelationalVerSyncDBInterface::PutLocalData(const std::vector &dataList, +int VirtualRelationalVerSyncDBInterface::PutLocalData(const std::vector &dataList, const std::string &tableName) { for (const auto &item : dataList) { - localData_[tableName].push_back(item); + localData_[tableName][item.logInfo.hashKey] = item; } return E_OK; } @@ -123,11 +116,21 @@ int VirtualRelationalVerSyncDBInterface::GetSyncData(QueryObject &query, } std::vector dataItemList; TableDataWithLog tableDataWithLog = {query.GetTableName(), {}}; - for (const auto &data : localData_[query.GetTableName()]) { - if (data.logInfo.timestamp >= timeRange.beginTime && data.logInfo.timestamp < timeRange.endTime) { - tableDataWithLog.dataList.push_back(data); + for (const auto &[hashKey, virtualData] : localData_[query.GetTableName()]) { + if (virtualData.logInfo.timestamp < timeRange.beginTime + || virtualData.logInfo.timestamp >= timeRange.endTime) { + continue; + } + RowDataWithLog rowData; + for (const auto &field : localFieldInfo_) { + DataValue dataValue; + (void)virtualData.objectData.GetDataValue(field.GetFieldName(), dataValue); + rowData.rowData.push_back(std::move(dataValue)); } + rowData.logInfo = virtualData.logInfo; + tableDataWithLog.dataList.push_back(rowData); } + int errCode = DataTransformer::TransformTableData(tableDataWithLog, localFieldInfo_, dataItemList); if (errCode != E_OK) { return errCode; @@ -210,9 +213,9 @@ std::vector VirtualRelationalVerSyncDBInterface::GetIdentifier() const void VirtualRelationalVerSyncDBInterface::GetMaxTimeStamp(TimeStamp &stamp) const { for (const auto &item : syncData_) { - for (const auto &rowDataWithLog : item.second) { - if (stamp < rowDataWithLog.logInfo.timestamp) { - stamp = rowDataWithLog.logInfo.timestamp; + for (const auto &[hashKey, virtualRowData] : item.second) { + if (stamp < virtualRowData.logInfo.timestamp) { + stamp = virtualRowData.logInfo.timestamp; } } } @@ -284,12 +287,27 @@ void VirtualRelationalVerSyncDBInterface::SetLocalFieldInfo(const std::vector &data) + std::vector &data) +{ + if (syncData_.find(tableName) == syncData_.end()) { + return -E_NOT_FOUND; + } + for (const auto &[hashKey, virtualRowData] : syncData_[tableName]) { + data.push_back(virtualRowData); + } + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetSyncData(const std::string &tableName, + const std::string hashKey, VirtualRowData &data) { if (syncData_.find(tableName) == syncData_.end()) { return -E_NOT_FOUND; } - data = syncData_[tableName]; + if (syncData_.find(hashKey) == syncData_.end()) { + return -E_NOT_FOUND; + } + data = syncData_[tableName][hashKey]; return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h index 318611909..e4f3dbd7a 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h @@ -22,6 +22,11 @@ #include "relational_schema_object.h" namespace DistributedDB { +struct VirtualRowData{ + LogInfo logInfo; + ObjectData objectData; +}; + class VirtualRelationalVerSyncDBInterface : public RelationalDBSyncInterface { public: VirtualRelationalVerSyncDBInterface() = default; @@ -30,7 +35,7 @@ public: int PutSyncDataWithQuery(const QueryObject &query, const std::vector &entries, const std::string &deviceName) override; - int PutLocalData(const std::vector &dataList, const std::string &tableName); + int PutLocalData(const std::vector &dataList, const std::string &tableName); RelationalSchemaObject GetSchemaInfo() const override; @@ -74,7 +79,9 @@ public: void SetLocalFieldInfo(const std::vector &localFieldInfo); - int GetAllSyncData(const std::string &tableName, std::vector &data); + int GetAllSyncData(const std::string &tableName, std::vector &data); + + int GetSyncData(const std::string &tableName, const std::string hashKey, VirtualRowData &data); int InterceptData(std::vector &entries, const std::string &sourceID, const std::string &targetID) const override @@ -94,8 +101,8 @@ public: private: mutable std::map, std::vector> metadata_; - std::map> syncData_; - mutable std::map> localData_; + std::map> syncData_; + mutable std::map> localData_; std::string schema_; RelationalSchemaObject schemaObj_; std::vector localFieldInfo_; -- Gitee From 4b5a82f7e93b37f4e34ebc5d5506306491c77949 Mon Sep 17 00:00:00 2001 From: zqq Date: Wed, 5 Jan 2022 17:05:01 +0800 Subject: [PATCH 08/38] remove unused include file Signed-off-by: zqq --- .../test/unittest/common/syncer/distributeddb_time_sync_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp index 745bb4df3..f002c34f5 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp @@ -19,7 +19,6 @@ #include "distributeddb_tools_unit_test.h" #include "isyncer.h" #include "single_ver_sync_state_machine.h" -#include "single_ver_sync_task_context.h" #include "sync_types.h" #include "virtual_single_ver_sync_db_Interface.h" #include "virtual_time_sync_communicator.h" -- Gitee From ae32b3769ee25f6078ab75f7e5002fee3092fd35 Mon Sep 17 00:00:00 2001 From: zqq Date: Wed, 5 Jan 2022 17:07:34 +0800 Subject: [PATCH 09/38] add tab Signed-off-by: zqq --- .../distributeddb/common/include/auto_launch.h | 4 ++-- .../distributeddb/common/src/auto_launch.cpp | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h index a275a5e44..5a2d9a4ac 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h @@ -143,9 +143,9 @@ private: static int GetAutoLaunchRelationProperties(const AutoLaunchParam ¶m, const std::shared_ptr &propertiesPtr); - static int OpenOneKvConnection(AutoLaunchItem &autoLaunchItem); + static int OpenKvConnection(AutoLaunchItem &autoLaunchItem); - static int OpenOneRelationalConnection(AutoLaunchItem &autoLaunchItem); + static int OpenRelationalConnection(AutoLaunchItem &autoLaunchItem); int RegisterLifeCycleCallback(AutoLaunchItem &autoLaunchItem, const std::string &identifier, bool isExt); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp index b39122471..ca5a026a3 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp @@ -500,12 +500,12 @@ int AutoLaunch::OpenOneConnection(AutoLaunchItem &autoLaunchItem) { LOGI("[AutoLaunch] GetOneConnection"); switch (autoLaunchItem.type) { - case DBType::DB_KV: - return OpenOneKvConnection(autoLaunchItem); - case DBType::DB_RELATION: - return OpenOneRelationalConnection(autoLaunchItem); - default: - return -E_INVALID_ARGS; + case DBType::DB_KV: + return OpenKvConnection(autoLaunchItem); + case DBType::DB_RELATION: + return OpenRelationalConnection(autoLaunchItem); + default: + return -E_INVALID_ARGS; } } @@ -984,7 +984,7 @@ int AutoLaunch::ExtAutoLaunchRequestCallBack(const std::string &identifier, Auto return E_OK; } -int AutoLaunch::OpenOneKvConnection(AutoLaunchItem &autoLaunchItem) +int AutoLaunch::OpenKvConnection(AutoLaunchItem &autoLaunchItem) { std::shared_ptr properties = std::static_pointer_cast(autoLaunchItem.propertiesPtr); @@ -999,7 +999,7 @@ int AutoLaunch::OpenOneKvConnection(AutoLaunchItem &autoLaunchItem) return errCode; } -int AutoLaunch::OpenOneRelationalConnection(AutoLaunchItem &autoLaunchItem) +int AutoLaunch::OpenRelationalConnection(AutoLaunchItem &autoLaunchItem) { std::shared_ptr properties = std::static_pointer_cast(autoLaunchItem.propertiesPtr); -- Gitee From e13f82e3dfcaf60571eb72a5018e6f00ecd904d6 Mon Sep 17 00:00:00 2001 From: zqq Date: Mon, 17 Jan 2022 14:56:02 +0800 Subject: [PATCH 10/38] add autoLaunch connection lifeCycle timer support Signed-off-by: zqq --- .../distributeddb/common/src/auto_launch.cpp | 1 + .../interfaces/src/relational/data_value.cpp | 13 +++ .../relational/relational_sync_able_storage.h | 9 +- .../src/relational_sync_able_storage.cpp | 15 +++ .../relational/sqlite_relational_store.cpp | 108 ++++++++++++++++++ .../relational/sqlite_relational_store.h | 14 +++ .../sqlite_relational_store_connection.cpp | 8 +- .../distributeddb/syncer/src/generic_syncer.h | 3 +- .../libs/distributeddb/test/BUILD.gn | 1 + 9 files changed, 168 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp index ca5a026a3..5a1b4969b 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp @@ -292,6 +292,7 @@ int AutoLaunch::RegisterObserver(AutoLaunchItem &autoLaunchItem, const std::stri LOGE("[AutoLaunch] autoLaunchItem.conn is nullptr"); return -E_INTERNAL_ERROR; } + LOGI("[AutoLaunch] RegisterObserver type=%d", autoLaunchItem.type); int errCode; Key key; KvDBObserverHandle *observerHandle = nullptr; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index 06718e615..3a9051c6e 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -187,6 +187,19 @@ DataValue &DataValue::operator=(const Blob &blob) return *this; } +int DataValue::Set(Blob *&blob) +{ + ResetValue(); + if (blob == nullptr || blob->GetSize() <= 0) { + LOGE("Transfer Blob to DataValue failed."); + return -E_INVALID_ARGS; + } + type_ = StorageType::STORAGE_TYPE_BLOB; + value_.blobPtr = blob; + blob = nullptr; + return E_OK; +} + DataValue &DataValue::operator=(const std::string &string) { (void)SetText(string); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index 378b72c98..ecf584fd9 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -92,8 +92,8 @@ public: int SchemaChanged(int notifyEvent) override; - int InterceptData(std::vector &entries, - const std::string &sourceID, const std::string &targetID) const override + int InterceptData(std::vector &entries,const std::string &sourceID, + const std::string &targetID) const override { return E_OK; } @@ -108,6 +108,9 @@ public: int RegisterSchemaChangedCallback(const std::function &callback) override; void NotifySchemaChanged(); + + void RegisterHeartBeatListener(const std::function &listener); + int GetCompressionAlgo(std::set &algorithmSet) const override; private: @@ -131,6 +134,8 @@ private: std::function onSchemaChanged_; mutable std::mutex onSchemaChangedMutex_; + std::function heartBeatListener_; + mutable std::mutex heartBeatMutex_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index 1f230f737..786272b49 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -18,6 +18,7 @@ #include "data_compression.h" #include "platform_specific.h" #include "generic_single_ver_kv_entry.h" +#include "runtime_context.h" namespace DistributedDB { #define CHECK_STORAGE_ENGINE do { \ @@ -93,6 +94,14 @@ void RelationalSyncAbleStorage::ReleaseHandle(SQLiteSingleVerRelationalStorageEx } StorageExecutor *databaseHandle = handle; storageEngine_->Recycle(databaseHandle); + std::function listener = nullptr; + { + std::lock_guard autoLock(heartBeatMutex_); + listener = heartBeatListener_; + } + if (listener) { + listener(); + } } // Get meta data associated with the given key. @@ -489,5 +498,11 @@ int RelationalSyncAbleStorage::GetCompressionAlgo(std::set &a DataCompression::GetCompressionAlgo(algorithmSet); return E_OK; } + +void RelationalSyncAbleStorage::RegisterHeartBeatListener(const std::function &listener) +{ + std::lock_guard autoLock(heartBeatMutex_); + heartBeatListener_ = listener; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 18b01a3de..c752fc858 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -27,6 +27,7 @@ namespace { constexpr const char *RELATIONAL_SCHEMA_KEY = "relational_schema"; constexpr const char *LOG_TABLE_VERSION_KEY = "log_table_versoin"; constexpr const char *LOG_TABLE_VERSION_1 = "1.0"; + constexpr int DEF_LIFE_CYCLE_TIME = 60000; // 60S } SQLiteRelationalStore::~SQLiteRelationalStore() @@ -424,5 +425,112 @@ int SQLiteRelationalStore::RemoveDeviceData(const std::string &device, const std ReleaseHandle(handle); return errCode; } + +int SQLiteRelationalStore::StopLifeCycleTimer() const +{ + auto runtimeCxt = RuntimeContext::GetInstance(); + if (runtimeCxt == nullptr) { + return -E_INVALID_ARGS; + } + if (lifeTimerId_ != 0) { + TimerId timerId = lifeTimerId_; + lifeTimerId_ = 0; + runtimeCxt->RemoveTimer(timerId, false); + } + return E_OK; +} + +int SQLiteRelationalStore::StartLifeCycleTimer(const DatabaseLifeCycleNotifier ¬ifier) const +{ + auto runtimeCxt = RuntimeContext::GetInstance(); + if (runtimeCxt == nullptr) { + return -E_INVALID_ARGS; + } + RefObject::IncObjRef(this); + TimerId timerId = 0; + int errCode = runtimeCxt->SetTimer(DEF_LIFE_CYCLE_TIME, + [this](TimerId id) -> int { + std::lock_guard lock(lifeCycleMutex_); + if (lifeCycleNotifier_) { + auto identifier = properties_.GetStringProp(DBProperties::IDENTIFIER_DATA, ""); + lifeCycleNotifier_(identifier); + } + return 0; + }, + [this]() { + int ret = RuntimeContext::GetInstance()->ScheduleTask([this]() { + RefObject::DecObjRef(this); + }); + if (ret != E_OK) { + LOGE("SQLiteSingleVerNaturalStore timer finalizer ScheduleTask, errCode %d", ret); + } + }, + timerId); + if (errCode != E_OK) { + lifeTimerId_ = 0; + LOGE("SetTimer failed:%d", errCode); + RefObject::DecObjRef(this); + return errCode; + } + + lifeCycleNotifier_ = notifier; + lifeTimerId_ = timerId; + return E_OK; +} + +int SQLiteRelationalStore::RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) +{ + int errCode; + { + std::lock_guard lock(lifeCycleMutex_); + if (!notifier) { + if (lifeTimerId_ == 0) { + return E_OK; + } + errCode = StopLifeCycleTimer(); + if (errCode != E_OK) { + LOGE("Stop the life cycle timer failed:%d", errCode); + } + return E_OK; + } + + if (lifeTimerId_ != 0) { + errCode = StopLifeCycleTimer(); + if (errCode != E_OK) { + LOGE("Stop the life cycle timer failed:%d", errCode); + } + } + errCode = StartLifeCycleTimer(notifier); + if (errCode != E_OK) { + LOGE("Register life cycle timer failed:%d", errCode); + } + } + auto listener = std::bind(&SQLiteRelationalStore::HeartBeat, this); + storageEngine_->RegisterHeartBeatListener(listener); + return errCode; +} + +void SQLiteRelationalStore::HeartBeat() const +{ + std::lock_guard lock(lifeCycleMutex_); + int errCode = ResetLifeCycleTimer(); + if (errCode != E_OK) { + LOGE("Heart beat for life cycle failed:%d", errCode); + } +} + +int SQLiteRelationalStore::ResetLifeCycleTimer() const +{ + if (lifeTimerId_ == 0) { + return E_OK; + } + auto lifeNotifier = lifeCycleNotifier_; + lifeCycleNotifier_ = nullptr; + int errCode = StopLifeCycleTimer(); + if (errCode != E_OK) { + LOGE("[Reset timer]Stop the life cycle timer failed:%d", errCode); + } + return StartLifeCycleTimer(lifeNotifier); +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 06ea31501..c4f984032 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -25,6 +25,7 @@ #include "isyncer.h" #include "sync_able_engine.h" #include "relational_sync_able_storage.h" +#include "runtime_context.h" namespace DistributedDB { class SQLiteRelationalStore : public IRelationalStore { @@ -55,6 +56,8 @@ public: int RemoveDeviceData(const std::string &device, const std::string &tableName); + int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier); + private: void ReleaseResources(); @@ -68,6 +71,11 @@ private: int CleanDistributedDeviceTable(); + int StopLifeCycleTimer() const; + int StartLifeCycleTimer(const DatabaseLifeCycleNotifier ¬ifier) const; + void HeartBeat() const; + int ResetLifeCycleTimer() const; + // use for sync Interactive std::unique_ptr syncEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv @@ -86,6 +94,12 @@ private: mutable std::mutex initalMutex_; bool isInitialized_ = false; + mutable std::mutex schemaMutex_; + //lifeCycle + mutable std::mutex lifeCycleMutex_; + mutable DatabaseLifeCycleNotifier lifeCycleNotifier_; + mutable TimerId lifeTimerId_; + }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index 7b4b66ed8..607b3f4ae 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -210,7 +210,13 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) int SQLiteRelationalStoreConnection::RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) { - return E_OK; + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[RelationalConnection] store is null, get executor failed!"); + return -E_INVALID_CONNECTION; + } + + return store->RegisterLifeCycleCallback(notifier); } } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h index 47bd27ff5..5c4048e1b 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h @@ -88,12 +88,13 @@ public: // Inner function, Used for subscribe sync int Sync(const InternalSyncParma ¶m); -protected: // Remote data changed callback virtual void RemoteDataChanged(const std::string &device) = 0; virtual void RemoteDeviceOffline(const std::string &device) = 0; +protected: + // trigger query auto sync or auto subscribe // trigger auto subscribe only when subscribe task is failed triggered by remote db opened // it won't be triggered again when subscribe task success diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index ab5fb8e10..337180528 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -243,6 +243,7 @@ ohos_source_set("src_file") { "unittest/common/interfaces/process_system_api_adapter_impl.cpp", "unittest/common/syncer/generic_virtual_device.cpp", "unittest/common/syncer/kv_virtual_device.cpp", + "unittest/common/syncer/relational_virtual_device.cpp", "unittest/common/syncer/virtual_communicator.cpp", "unittest/common/syncer/virtual_communicator_aggregator.cpp", "unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp", -- Gitee From aeb3f98f6bfd1331632d39fecae9c8b2c74f774e Mon Sep 17 00:00:00 2001 From: zqq Date: Tue, 18 Jan 2022 11:44:50 +0800 Subject: [PATCH 11/38] modify print log msg Signed-off-by: zqq --- .../common/include/auto_launch.h | 2 +- .../common/include/db_constant.h | 2 ++ .../distributeddb/common/src/auto_launch.cpp | 32 +++++++++---------- .../relational/sqlite_relational_store.cpp | 3 +- .../sqlite_single_ver_natural_store.cpp | 3 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h index 5a2d9a4ac..aec62eb24 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h @@ -86,7 +86,7 @@ private: int EnableKvStoreAutoLaunchParmCheck(AutoLaunchItem &autoLaunchItem, const std::string &identifier); - int GetConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std::string &identifier); + int GetKVConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std::string &identifier); static int OpenOneConnection(AutoLaunchItem &autoLaunchItem); diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index ca7403d54..95e7ae6f0 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -122,6 +122,8 @@ public: static constexpr uint64_t MAX_LOG_SIZE_LOW = 0x400000ULL; // 4MB static constexpr uint64_t MAX_LOG_SIZE_DEFAULT = 0x40000000ULL; // 1GB + static constexpr int DEF_LIFE_CYCLE_TIME = 60000; // 60S + // For relational static const std::string RELATIONAL_PREFIX; static const std::string TIMESTAMP_ALIAS; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp index 5a1b4969b..79b8d04d1 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp @@ -146,7 +146,7 @@ int AutoLaunch::EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLa LOGE("[AutoLaunch] EnableKvStoreAutoLaunch failed errCode:%d", errCode); return errCode; } - errCode = GetConnectionInEnable(autoLaunchItem, identifier); + errCode = GetKVConnectionInEnable(autoLaunchItem, identifier); if (errCode == E_OK) { LOGI("[AutoLaunch] EnableKvStoreAutoLaunch ok"); } else { @@ -155,21 +155,21 @@ int AutoLaunch::EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLa return errCode; } -int AutoLaunch::GetConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std::string &identifier) +int AutoLaunch::GetKVConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std::string &identifier) { - LOGI("[AutoLaunch] GetConnectionInEnable"); + LOGI("[AutoLaunch] GetKVConnectionInEnable"); int errCode; std::shared_ptr properties = std::static_pointer_cast(autoLaunchItem.propertiesPtr); autoLaunchItem.conn = KvDBManager::GetDatabaseConnection(*properties, errCode, false); if (errCode == -E_ALREADY_OPENED) { - LOGI("[AutoLaunch] GetConnectionInEnable user already getkvstore by self"); + LOGI("[AutoLaunch] GetKVConnectionInEnable user already getkvstore by self"); std::lock_guard autoLock(dataLock_); autoLaunchItemMap_[identifier].state = AutoLaunchItemState::IDLE; return E_OK; } if (autoLaunchItem.conn == nullptr) { - LOGE("[AutoLaunch] GetConnectionInEnable GetDatabaseConnection errCode:%d", errCode); + LOGE("[AutoLaunch] GetKVConnectionInEnable GetDatabaseConnection errCode:%d", errCode); std::lock_guard autoLock(dataLock_); autoLaunchItemMap_.erase(identifier); return errCode; @@ -180,11 +180,11 @@ int AutoLaunch::GetConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std: isEmpty = onlineDevices_.empty(); } if (isEmpty) { - LOGI("[AutoLaunch] GetConnectionInEnable no online device, ReleaseDatabaseConnection"); + LOGI("[AutoLaunch] GetKVConnectionInEnable no online device, ReleaseDatabaseConnection"); IKvDBConnection *kvConn = static_cast(autoLaunchItem.conn); errCode = KvDBManager::ReleaseDatabaseConnection(kvConn); if (errCode != E_OK) { - LOGE("[AutoLaunch] GetConnectionInEnable ReleaseDatabaseConnection failed errCode:%d", errCode); + LOGE("[AutoLaunch] GetKVConnectionInEnable ReleaseDatabaseConnection failed errCode:%d", errCode); std::lock_guard autoLock(dataLock_); autoLaunchItemMap_.erase(identifier); return errCode; @@ -199,9 +199,9 @@ int AutoLaunch::GetConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std: autoLaunchItemMap_[identifier].state = AutoLaunchItemState::IDLE; autoLaunchItemMap_[identifier].conn = autoLaunchItem.conn; autoLaunchItemMap_[identifier].observerHandle = autoLaunchItem.observerHandle; - LOGI("[AutoLaunch] GetConnectionInEnable RegisterObserverAndLifeCycleCallback ok"); + LOGI("[AutoLaunch] GetKVConnectionInEnable RegisterObserverAndLifeCycleCallback ok"); } else { - LOGE("[AutoLaunch] GetConnectionInEnable RegisterObserverAndLifeCycleCallback err, do CloseConnection"); + LOGE("[AutoLaunch] GetKVConnectionInEnable RegisterObserverAndLifeCycleCallback err, do CloseConnection"); TryCloseConnection(autoLaunchItem); // do nothing if failed std::lock_guard autoLock(dataLock_); autoLaunchItemMap_.erase(identifier); @@ -1065,24 +1065,24 @@ void AutoLaunch::TryCloseKvConnection(AutoLaunchItem &autoLaunchItem) { LOGI("[AutoLaunch] TryCloseKvConnection"); if (autoLaunchItem.conn == nullptr) { - LOGI("[AutoLaunch] TryCloseConnection conn is nullptr, do nothing"); + LOGI("[AutoLaunch] TryCloseKvConnection conn is nullptr, do nothing"); return; } IKvDBConnection *kvConn = static_cast(autoLaunchItem.conn); int errCode = kvConn->RegisterLifeCycleCallback(nullptr); if (errCode != E_OK) { - LOGE("[AutoLaunch] TryCloseConnection RegisterLifeCycleCallback failed errCode:%d", errCode); + LOGE("[AutoLaunch] TryCloseKvConnection RegisterLifeCycleCallback failed errCode:%d", errCode); } if (autoLaunchItem.observerHandle != nullptr) { errCode = kvConn->UnRegisterObserver(autoLaunchItem.observerHandle); if (errCode != E_OK) { - LOGE("[AutoLaunch] TryCloseConnection UnRegisterObserver failed errCode:%d", errCode); + LOGE("[AutoLaunch] TryCloseKvConnection UnRegisterObserver failed errCode:%d", errCode); } autoLaunchItem.observerHandle = nullptr; } errCode = KvDBManager::ReleaseDatabaseConnection(kvConn); if (errCode != E_OK) { - LOGE("[AutoLaunch] TryCloseConnection ReleaseDatabaseConnection failed errCode:%d", errCode); + LOGE("[AutoLaunch] TryCloseKvConnection ReleaseDatabaseConnection failed errCode:%d", errCode); } } @@ -1090,17 +1090,17 @@ void AutoLaunch::TryCloseRelationConnection(AutoLaunchItem &autoLaunchItem) { LOGI("[AutoLaunch] TryCloseRelationConnection"); if (autoLaunchItem.conn == nullptr) { - LOGI("[AutoLaunch] TryCloseConnection conn is nullptr, do nothing"); + LOGI("[AutoLaunch] TryCloseRelationConnection conn is nullptr, do nothing"); return; } RelationalStoreConnection *rdbConn = static_cast(autoLaunchItem.conn); int errCode = rdbConn->RegisterLifeCycleCallback(nullptr); if (errCode != E_OK) { - LOGE("[AutoLaunch] TryCloseConnection RegisterLifeCycleCallback failed errCode:%d", errCode); + LOGE("[AutoLaunch] TryCloseRelationConnection RegisterLifeCycleCallback failed errCode:%d", errCode); } errCode = rdbConn->Close(); if (errCode != E_OK) { - LOGE("[AutoLaunch] TryCloseConnection close connection failed errCode:%d", errCode); + LOGE("[AutoLaunch] TryCloseRelationConnection close connection failed errCode:%d", errCode); } } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index c752fc858..f28ce1b5b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -27,7 +27,6 @@ namespace { constexpr const char *RELATIONAL_SCHEMA_KEY = "relational_schema"; constexpr const char *LOG_TABLE_VERSION_KEY = "log_table_versoin"; constexpr const char *LOG_TABLE_VERSION_1 = "1.0"; - constexpr int DEF_LIFE_CYCLE_TIME = 60000; // 60S } SQLiteRelationalStore::~SQLiteRelationalStore() @@ -448,7 +447,7 @@ int SQLiteRelationalStore::StartLifeCycleTimer(const DatabaseLifeCycleNotifier & } RefObject::IncObjRef(this); TimerId timerId = 0; - int errCode = runtimeCxt->SetTimer(DEF_LIFE_CYCLE_TIME, + int errCode = runtimeCxt->SetTimer(DBConstant::DEF_LIFE_CYCLE_TIME, [this](TimerId id) -> int { std::lock_guard lock(lifeCycleMutex_); if (lifeCycleNotifier_) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp index 1110153a5..797e9a93e 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp @@ -42,7 +42,6 @@ namespace DistributedDB { } while (0) namespace { - constexpr int DEF_LIFE_CYCLE_TIME = 60000; // 60s constexpr int WAIT_DELEGATE_CALLBACK_TIME = 100; const std::string CREATE_DB_TIME = "createDBTime"; @@ -172,7 +171,7 @@ SQLiteSingleVerNaturalStore::SQLiteSingleVerNaturalStore() isReadOnly_(false), lifeCycleNotifier_(nullptr), lifeTimerId_(0), - autoLifeTime_(DEF_LIFE_CYCLE_TIME), + autoLifeTime_(DBConstant::DEF_LIFE_CYCLE_TIME), createDBTime_(0), dataInterceptor_(nullptr), maxLogSize_(DBConstant::MAX_LOG_SIZE_DEFAULT) -- Gitee From 1a782c4fdd644b3126f58c7cf4a9f44ef63f5632 Mon Sep 17 00:00:00 2001 From: zqq Date: Wed, 19 Jan 2022 17:12:25 +0800 Subject: [PATCH 12/38] abort when RegisterLifeCycleCallback failed in rdb Signed-off-by: zqq --- .../storage/src/sqlite/relational/sqlite_relational_store.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index f28ce1b5b..b516ab606 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -497,11 +497,13 @@ int SQLiteRelationalStore::RegisterLifeCycleCallback(const DatabaseLifeCycleNoti errCode = StopLifeCycleTimer(); if (errCode != E_OK) { LOGE("Stop the life cycle timer failed:%d", errCode); + return errCode; } } errCode = StartLifeCycleTimer(notifier); if (errCode != E_OK) { LOGE("Register life cycle timer failed:%d", errCode); + return errCode; } } auto listener = std::bind(&SQLiteRelationalStore::HeartBeat, this); -- Gitee From 12816645d5375ac8d9675ec8947841df360e7e8e Mon Sep 17 00:00:00 2001 From: lianhuix Date: Fri, 14 Jan 2022 11:50:55 +0800 Subject: [PATCH 13/38] Add routines ext Signed-off-by: lianhuix --- .../relational/relational_store_sqlite_ext.h | 18 +- .../relational_store_sqlite_ext.cpp | 373 ++++++++++++++++-- 2 files changed, 352 insertions(+), 39 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h index ab7ed7117..72421bc20 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h @@ -18,13 +18,15 @@ #define SQLITE3_HW_EXPORT_SYMBOLS +// using the "sqlite3sym.h" in OHOS +#ifndef USE_SQLITE_SYMBOLS #include "sqlite3.h" -#include "types.h" +#else +#include "sqlite3sym.h" +#endif // We extend the original purpose of the "sqlite3ext.h". struct sqlite3_api_routines_relational { - int (*close)(sqlite3*); - int (*close_v2)(sqlite3*); int (*open)(const char *, sqlite3 **); int (*open16)(const void *, sqlite3 **); int (*open_v2)(const char *, sqlite3 **, int, const char *); @@ -32,16 +34,6 @@ struct sqlite3_api_routines_relational { extern const struct sqlite3_api_routines_relational *sqlite3_export_relational_symbols; -#ifdef sqlite3_close -#undef sqlite3_close -#endif -#define sqlite3_close sqlite3_export_relational_symbols->close - -#ifdef sqlite3_close_v2 -#undef sqlite3_close_v2 -#endif -#define sqlite3_close_v2 sqlite3_export_relational_symbols->close_v2 - #ifdef sqlite3_open #undef sqlite3_open #endif diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp index 70209f45d..8cdefb026 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp @@ -13,47 +13,374 @@ * limitations under the License. */ #ifdef RELATIONAL_STORE -#include "sqlite_utils.h" -#include "log_print.h" -using namespace DistributedDB; -// namespace DistributedDB { -// We extend the original purpose of the "sqlite3ext.h". -SQLITE_API int sqlite3_close_relational(sqlite3* db) + +#include +#include +#include +#include +#include + +// using the "sqlite3sym.h" in OHOS +#ifndef USE_SQLITE_SYMBOLS +#include "sqlite3.h" +#else +#include "sqlite3sym.h" +#endif + +namespace { +constexpr int E_OK = 0; +constexpr int E_ERROR = 1; + +class ValueHashCalc { +public: + ValueHashCalc() {}; + ~ValueHashCalc() + { + if (context_ != nullptr) { + delete context_; + context_ = nullptr; + } + } + + int Initialize() + { + context_ = new (std::nothrow) SHA256_CTX; + if (context_ == nullptr) { + return -E_ERROR; + } + + int errCode = SHA256_Init(context_); + if (errCode == 0) { + return -E_ERROR; + } + return E_OK; + } + + int Update(const std::vector &value) + { + if (context_ == nullptr) { + return -E_ERROR; + } + int errCode = SHA256_Update(context_, value.data(), value.size()); + if (errCode == 0) { + return -E_ERROR; + } + return E_OK; + } + + int GetResult(std::vector &value) + { + if (context_ == nullptr) { + return -E_ERROR; + } + + value.resize(SHA256_DIGEST_LENGTH); + int errCode = SHA256_Final(value.data(), context_); + if (errCode == 0) { + return -E_ERROR; + } + + return E_OK; + } + +private: + SHA256_CTX *context_ = nullptr; +}; + + +const uint64_t MULTIPLES_BETWEEN_SECONDS_AND_MICROSECONDS = 1000000; + +using TimeStamp = uint64_t; +using TimeOffset = int64_t; + +class TimeHelper { +public: + constexpr static int64_t BASE_OFFSET = 10000LL * 365LL * 24LL * 3600LL * 1000LL * 1000LL * 10L; // 10000 year 100ns + + constexpr static int64_t MAX_VALID_TIME = BASE_OFFSET * 2; // 20000 year 100ns + + static const uint64_t TO_100_NS = 10; // 1us to 100ns + + static const uint64_t MS_TO_100_NS = 10000; // 1ms to 100ns + + static const TimeStamp INVALID_TIMESTAMP = 0; + + // Get current system time + static TimeStamp GetSysCurrentTime() + { + uint64_t curTime = 0; + std::lock_guard lock(systemTimeLock_); + int errCode = GetCurrentSysTimeInMicrosecond(curTime); + if (errCode != E_OK) { + return INVALID_TIMESTAMP; + } + + // If GetSysCurrentTime in 1us, we need increase the currentIncCount_ + if (curTime == lastSystemTimeUs_) { + // if the currentIncCount_ has been increased MAX_INC_COUNT, keep the currentIncCount_ + if (currentIncCount_ < MAX_INC_COUNT) { + currentIncCount_++; + } + } else { + lastSystemTimeUs_ = curTime; + currentIncCount_ = 0; + } + return (curTime * TO_100_NS) + currentIncCount_; // Currently TimeStamp is uint64_t + } + + // Init the TimeHelper + static void Initialize(TimeStamp maxTimeStamp) + { + std::lock_guard lock(lastLocalTimeLock_); + if (lastSystemTimeUs_ < maxTimeStamp) { + lastSystemTimeUs_ = maxTimeStamp; + } + } + + static TimeStamp GetTime(TimeOffset TimeOffset) + { + std::lock_guard lock(lastLocalTimeLock_); + TimeStamp currentSysTime = GetSysCurrentTime(); + TimeStamp currentLocalTime = currentSysTime + TimeOffset; + if (currentLocalTime <= lastLocalTime_ || currentLocalTime > MAX_VALID_TIME) { + lastLocalTime_++; + currentLocalTime = lastLocalTime_; + } else { + lastLocalTime_ = currentLocalTime; + } + return currentLocalTime; + } + +private: + static int GetCurrentSysTimeInMicrosecond(uint64_t &outTime) + { + struct timeval rawTime; + int errCode = gettimeofday(&rawTime, nullptr); + if (errCode < 0) { + return -E_ERROR; + } + outTime = static_cast(rawTime.tv_sec) * MULTIPLES_BETWEEN_SECONDS_AND_MICROSECONDS + + static_cast(rawTime.tv_usec); + return E_OK; + } + + static std::mutex systemTimeLock_; + static TimeStamp lastSystemTimeUs_; + static TimeStamp currentIncCount_; + static const uint64_t MAX_INC_COUNT = 9; // last bit from 0-9 + + static TimeStamp lastLocalTime_; + static std::mutex lastLocalTimeLock_; +}; + +std::mutex TimeHelper::systemTimeLock_; +TimeStamp TimeHelper::lastSystemTimeUs_ = 0; +TimeStamp TimeHelper::currentIncCount_ = 0; +TimeStamp TimeHelper::lastLocalTime_ = 0; +std::mutex TimeHelper::lastLocalTimeLock_; + +struct TransactFunc { + void (*xFunc)(sqlite3_context*, int, sqlite3_value**) = nullptr; + void (*xStep)(sqlite3_context*, int, sqlite3_value**) = nullptr; + void (*xFinal)(sqlite3_context*) = nullptr; + void(*xDestroy)(void*) = nullptr; +}; + +int RegisterFunction(sqlite3 *db, const std::string &funcName, int nArg, void *uData, TransactFunc &func) { - ::LOGD("### my sqlite3_close_relational!"); - return sqlite3_close(db); + if (db == nullptr) { + return -E_ERROR; + } + return sqlite3_create_function_v2(db, funcName.c_str(), nArg, SQLITE_UTF8 | SQLITE_DETERMINISTIC, uData, + func.xFunc, func.xStep, func.xFinal, func.xDestroy); } -SQLITE_API int sqlite3_close_v2_relational(sqlite3* db) +int CalcValueHash(const std::vector &value, std::vector &hashValue) { - ::LOGD("### my sqlite3_close_v2_relational!"); - return sqlite3_close_v2(db); + ValueHashCalc hashCalc; + int errCode = hashCalc.Initialize(); + if (errCode != E_OK) { + return -E_ERROR; + } + + errCode = hashCalc.Update(value); + if (errCode != E_OK) { + return -E_ERROR; + } + + errCode = hashCalc.GetResult(hashValue); + if (errCode != E_OK) { + return -E_ERROR; + } + + return E_OK; +} + +void CalcHashKey(sqlite3_context *ctx, int argc, sqlite3_value **argv) +{ + // 1 means that the function only needs one parameter, namely key + if (ctx == nullptr || argc != 1 || argv == nullptr) { + return; + } + auto keyBlob = static_cast(sqlite3_value_blob(argv[0])); + if (keyBlob == nullptr) { + sqlite3_result_error(ctx, "Parameters is invalid.", -1); + return; + } + int blobLen = sqlite3_value_bytes(argv[0]); + std::vector value(keyBlob, keyBlob + blobLen); + std::vector hashValue; + int errCode = CalcValueHash(value, hashValue); + if (errCode != E_OK) { + sqlite3_result_error(ctx, "Get hash value error.", -1); + return; + } + sqlite3_result_blob(ctx, hashValue.data(), hashValue.size(), SQLITE_TRANSIENT); + return; +} + +int RegisterCalcHash(sqlite3 *db) +{ + TransactFunc func; + func.xFunc = &CalcHashKey; + return RegisterFunction(db, "calc_hash", 1, nullptr, func); +} + +void GetSysTime(sqlite3_context *ctx, int argc, sqlite3_value **argv) +{ + if (ctx == nullptr || argc != 1 || argv == nullptr) { // 1: function need one parameter + return; + } + int timeOffset = static_cast(sqlite3_value_int64(argv[0])); + sqlite3_result_int64(ctx, (sqlite3_int64)TimeHelper::GetTime(timeOffset)); +} + +int RegisterGetSysTime(sqlite3 *db) +{ + TransactFunc func; + func.xFunc = &GetSysTime; + return RegisterFunction(db, "get_sys_time", 1, nullptr, func); +} + +int ResetStatement(sqlite3_stmt *&stmt) +{ + if (stmt == nullptr || sqlite3_finalize(stmt) != SQLITE_OK) { + return -E_ERROR; + } + stmt = nullptr; + return E_OK; +} + +int GetStatement(sqlite3 *db, const std::string &sql, sqlite3_stmt *&stmt) +{ + int errCode = sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, nullptr); + if (errCode != SQLITE_OK) { + (void)ResetStatement(stmt); + return -E_ERROR; + } + return E_OK; +} + +int StepWithRetry(sqlite3_stmt *stmt) +{ + if (stmt == nullptr) { + return -E_ERROR; + } + int errCode = sqlite3_step(stmt); + if (errCode != SQLITE_DONE && errCode != SQLITE_ROW) { + return -E_ERROR; + } + return errCode; +} + +int GetColumnTestValue(sqlite3_stmt *stmt, int index, std::string &value) +{ + if (stmt == nullptr) { + return -E_ERROR; + } + const unsigned char *val = sqlite3_column_text(stmt, index); + value = (val != nullptr) ? std::string(reinterpret_cast(val)) : std::string(); + return E_OK; +} + +int GetCurrentMaxTimeStamp(sqlite3 *db, TimeStamp &maxTimestamp) +{ + if (db == nullptr) { + return -E_ERROR; + } + std::string checkTableSql = "SELECT name FROM sqlite_master WHERE type = 'table' AND " \ + "name LIKE 'naturalbase_rdb_aux_%_log';"; + sqlite3_stmt *checkTableStmt = nullptr; + int errCode = GetStatement(db, checkTableSql, checkTableStmt); + if (errCode != E_OK) { + return -E_ERROR; + } + while ((errCode = StepWithRetry(checkTableStmt)) != SQLITE_DONE) { + std::string logTablename; + GetColumnTestValue(checkTableStmt, 0, logTablename); + if (logTablename.empty()) { + continue; + } + + std::string getMaxTimestampSql = "SELECT MAX(timestamp) FROM " + logTablename + ";"; + sqlite3_stmt *getTimeStmt = nullptr; + errCode = GetStatement(db, getMaxTimestampSql, getTimeStmt); + if (errCode != E_OK) { + continue; + } + errCode = StepWithRetry(getTimeStmt); + if (errCode != SQLITE_ROW) { + ResetStatement(getTimeStmt); + continue; + } + TimeStamp tableMaxTimestamp = sqlite3_column_int64(getTimeStmt, 0); + maxTimestamp = (maxTimestamp > tableMaxTimestamp) ? maxTimestamp : tableMaxTimestamp; + ResetStatement(getTimeStmt); + } + ResetStatement(checkTableStmt); + return E_OK; +} } SQLITE_API int sqlite3_open_relational(const char *filename, sqlite3 **ppDb) { int err = sqlite3_open(filename, ppDb); - DistributedDB::SQLiteUtils::RegisterCalcHash(*ppDb); - DistributedDB::SQLiteUtils::RegisterGetSysTime(*ppDb); - ::LOGD("### my sqlite3_open!"); + if (err != SQLITE_OK) { + return err; + } + TimeStamp currentMaxTimestamp = 0; + (void)GetCurrentMaxTimeStamp(*ppDb, currentMaxTimestamp); + TimeHelper::Initialize(currentMaxTimestamp); + RegisterCalcHash(*ppDb); + RegisterGetSysTime(*ppDb); return err; } SQLITE_API int sqlite3_open16_relational(const void *filename, sqlite3 **ppDb) { int err = sqlite3_open16(filename, ppDb); - DistributedDB::SQLiteUtils::RegisterCalcHash(*ppDb); - DistributedDB::SQLiteUtils::RegisterGetSysTime(*ppDb); - ::LOGD("### my sqlite3_open16!"); + if (err != SQLITE_OK) { + return err; + } + TimeStamp currentMaxTimestamp = 0; + (void)GetCurrentMaxTimeStamp(*ppDb, currentMaxTimestamp); + TimeHelper::Initialize(currentMaxTimestamp); + RegisterCalcHash(*ppDb); + RegisterGetSysTime(*ppDb); return err; } SQLITE_API int sqlite3_open_v2_relational(const char *filename, sqlite3 **ppDb, int flags, const char *zVfs) { int err = sqlite3_open_v2(filename, ppDb, flags, zVfs); - DistributedDB::SQLiteUtils::RegisterCalcHash(*ppDb); - DistributedDB::SQLiteUtils::RegisterGetSysTime(*ppDb); - ::LOGD("### my sqlite3_open_v2!"); + if (err != SQLITE_OK) { + return err; + } + TimeStamp currentMaxTimestamp = 0; + (void)GetCurrentMaxTimeStamp(*ppDb, currentMaxTimestamp); + TimeHelper::Initialize(currentMaxTimestamp); + RegisterCalcHash(*ppDb); + RegisterGetSysTime(*ppDb); return err; } @@ -68,8 +395,6 @@ SQLITE_API int sqlite3_open_v2_relational(const char *filename, sqlite3 **ppDb, #endif struct sqlite3_api_routines_relational { - int (*close)(sqlite3*); - int (*close_v2)(sqlite3*); int (*open)(const char *, sqlite3 **); int (*open16)(const void *, sqlite3 **); int (*open_v2)(const char *, sqlite3 **, int, const char *); @@ -78,14 +403,10 @@ struct sqlite3_api_routines_relational { typedef struct sqlite3_api_routines_relational sqlite3_api_routines_relational; static const sqlite3_api_routines_relational sqlite3HwApis = { #ifdef SQLITE_DISTRIBUTE_RELATIONAL - sqlite3_close_relational, - sqlite3_close_v2_relational, sqlite3_open_relational, sqlite3_open16_relational, sqlite3_open_v2_relational #else - 0, - 0, 0, 0, 0 -- Gitee From c5baf5de9d3a8a6cacabe96cc2b7749303b26665 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 18 Jan 2022 16:54:41 +0800 Subject: [PATCH 14/38] Remove RELATIONAL_STORE macro in routines Signed-off-by: lianhuix --- .../include/relational/relational_store_sqlite_ext.h | 2 -- .../src/relational/relational_store_sqlite_ext.cpp | 5 ----- 2 files changed, 7 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h index 72421bc20..59f4d5342 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h @@ -14,7 +14,6 @@ */ #ifndef RELATIONAL_STORE_EXT_H #define RELATIONAL_STORE_EXT_H -#ifdef RELATIONAL_STORE #define SQLITE3_HW_EXPORT_SYMBOLS @@ -49,5 +48,4 @@ extern const struct sqlite3_api_routines_relational *sqlite3_export_relational_s #endif #define sqlite3_open_v2 sqlite3_export_relational_symbols->open_v2 -#endif #endif // RELATIONAL_STORE_EXT_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp index 8cdefb026..f5409e1ef 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp @@ -12,7 +12,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifdef RELATIONAL_STORE #include #include @@ -101,8 +100,6 @@ public: static const uint64_t TO_100_NS = 10; // 1us to 100ns - static const uint64_t MS_TO_100_NS = 10000; // 1ms to 100ns - static const TimeStamp INVALID_TIMESTAMP = 0; // Get current system time @@ -414,6 +411,4 @@ static const sqlite3_api_routines_relational sqlite3HwApis = { }; EXPORT_SYMBOLS const sqlite3_api_routines_relational *sqlite3_export_relational_symbols = &sqlite3HwApis; -#endif - #endif \ No newline at end of file -- Gitee From d40b54f82685d4055877208b0333ffa6fb53bc30 Mon Sep 17 00:00:00 2001 From: zqq Date: Mon, 17 Jan 2022 18:48:26 +0800 Subject: [PATCH 15/38] 1.support erase rdb watermark 2.split sync_task_context to rdb and kv 3.parcel write bool as uint8 Signed-off-by: zqq --- .../libs/distributeddb/BUILD.gn | 2 + .../distributeddb/common/include/parcel.h | 3 + .../libs/distributeddb/common/src/parcel.cpp | 44 +++++++------- .../include/single_ver_kvdb_sync_interface.h | 4 -- .../storage/src/data_transformer.cpp | 56 +++++++++--------- .../storage/src/sync_able_engine.cpp | 5 +- .../storage/src/sync_able_engine.h | 2 +- .../distributeddb/syncer/include/isyncer.h | 3 +- .../syncer/include/syncer_proxy.h | 3 +- .../distributeddb/syncer/src/meta_data.cpp | 8 +-- .../libs/distributeddb/syncer/src/meta_data.h | 4 +- .../syncer/src/multi_ver_syncer.cpp | 3 +- .../syncer/src/multi_ver_syncer.h | 3 +- .../src/query_sync_water_mark_helper.cpp | 6 +- .../syncer/src/query_sync_water_mark_helper.h | 2 +- .../syncer/src/single_ver_data_sync.cpp | 58 ++++++------------- .../syncer/src/single_ver_data_sync.h | 3 - .../src/single_ver_kv_sync_task_context.cpp | 36 ++++++++++++ .../src/single_ver_kv_sync_task_context.h | 36 ++++++++++++ .../syncer/src/single_ver_kv_syncer.cpp | 11 +--- .../syncer/src/single_ver_kv_syncer.h | 3 - ...ingle_ver_relational_sync_task_context.cpp | 53 +++++++++++++++++ .../single_ver_relational_sync_task_context.h | 43 ++++++++++++++ .../src/single_ver_relational_syncer.cpp | 20 +++++-- .../syncer/src/single_ver_relational_syncer.h | 5 +- .../syncer/src/single_ver_sync_engine.cpp | 17 +++++- .../syncer/src/single_ver_sync_task_context.h | 12 ++-- .../syncer/src/single_ver_syncer.cpp | 16 +++++ .../syncer/src/single_ver_syncer.h | 6 ++ .../distributeddb/syncer/src/sync_engine.h | 2 +- .../syncer/src/sync_operation.cpp | 1 - .../distributeddb/syncer/src/syncer_proxy.cpp | 8 ++- .../libs/distributeddb/test/BUILD.gn | 2 + .../distributeddb_ability_sync_test.cpp | 6 +- ...ributeddb_single_ver_msg_schedule_test.cpp | 15 ++--- .../syncer/distributeddb_time_sync_test.cpp | 3 +- .../common/syncer/generic_virtual_device.cpp | 7 ++- ...rtual_relational_ver_sync_db_interface.cpp | 10 ++-- 38 files changed, 354 insertions(+), 167 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.h create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.h diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index 2ccb01575..29250daa3 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -224,6 +224,8 @@ ohos_shared_library("distributeddb") { "syncer/src/single_ver_sync_state_machine.cpp", "syncer/src/single_ver_sync_target.cpp", "syncer/src/single_ver_sync_task_context.cpp", + "syncer/src/single_ver_kv_sync_task_context.cpp", + "syncer/src/single_ver_relational_sync_task_context.cpp", "syncer/src/single_ver_syncer.cpp", "syncer/src/subscribe_manager.cpp", "syncer/src/sync_engine.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/common/include/parcel.h b/services/distributeddataservice/libs/distributeddb/common/include/parcel.h index 7658f860d..4d67976c7 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/parcel.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/parcel.h @@ -40,6 +40,8 @@ public: uint32_t ReadBool(bool &data); int WriteInt(int data); uint32_t ReadInt(int &val); + int WriteUInt8(uint8_t data); + uint32_t ReadUInt8(uint8_t &val); int WriteDouble(double data); uint32_t ReadDouble(double &val); int WriteInt64(int64_t data); @@ -140,6 +142,7 @@ public: void EightByteAlign(); // Avoid reading a single data type across 8 bytes static uint32_t GetBoolLen(); static uint32_t GetIntLen(); + static uint32_t GetUInt8Len(); static uint32_t GetUInt32Len(); static uint32_t GetUInt64Len(); static uint32_t GetInt64Len(); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp b/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp index e02f8685d..794a07116 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp @@ -48,32 +48,16 @@ bool Parcel::IsError() const int Parcel::WriteBool(bool data) { - const uint32_t boolLen = GetBoolLen(); - if (isError_ || parcelLen_ + boolLen > totalLen_) { - isError_ = true; - return -E_PARSE_FAIL; - } - errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &data, sizeof(bool)); - if (errCode != EOK) { - isError_ = true; - return -E_SECUREC_ERROR; - } - bufPtr_ += boolLen; - parcelLen_ += boolLen; - return E_OK; + uint8_t value = data ? 1 : 0; + return WriteUInt8(value); } uint32_t Parcel::ReadBool(bool &val) { - const uint32_t boolLen = GetBoolLen(); - if (isError_ || parcelLen_ + boolLen > totalLen_) { - isError_ = true; - return -E_PARSE_FAIL; - } - val = *(reinterpret_cast(bufPtr_)); - bufPtr_ += boolLen; - parcelLen_ += boolLen; - return boolLen; + uint8_t intVal = 0; + uint32_t len = ReadUInt8(intVal); + val = intVal == 1 ? true : false; + return len; } int Parcel::WriteInt(int32_t data) @@ -86,6 +70,15 @@ uint32_t Parcel::ReadInt(int32_t &val) return ReadInteger(val); } +int Parcel::WriteUInt8(uint8_t data) +{ + return WriteInteger(data); +} + +uint32_t Parcel::ReadUInt8(uint8_t &val) +{ + return ReadInteger(val); +} int Parcel::WriteDouble(double data) { @@ -434,7 +427,12 @@ uint32_t Parcel::ReadBlob(char *buffer, uint32_t bufLen) uint32_t Parcel::GetBoolLen() { - return GetEightByteAlign(sizeof(bool)); + return GetUInt8Len(); +} + +uint32_t Parcel::GetUInt8Len() +{ + return sizeof(uint8_t); } uint32_t Parcel::GetIntLen() diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h index 1cca304b6..fe56452bb 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h @@ -33,10 +33,6 @@ public: ~SingleVerKvDBSyncInterface() override = default; virtual SchemaObject GetSchemaInfo() const = 0; - - virtual bool CheckCompatible(const std::string &schema) const = 0; - - virtual int GetCompressionAlgo(std::set &algorithmSet) const = 0; }; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp index f55194374..eb30a61f1 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -120,16 +120,11 @@ uint32_t DataTransformer::CalDataValueLength(const DataValue &dataValue) uint32_t length = 0; switch (dataValue.GetType()) { case StorageType::STORAGE_TYPE_BLOB: + case StorageType::STORAGE_TYPE_TEXT: (void)dataValue.GetBlobLength(length); length = Parcel::GetEightByteAlign(length); length += Parcel::GetUInt32Len(); // record data length break; - case StorageType::STORAGE_TYPE_TEXT: { - std::string str; - (void)dataValue.GetText(str); - length = Parcel::GetStringLen(str); - break; - } default: break; } @@ -224,24 +219,6 @@ int DeSerializeDoubleValue(DataValue &dataValue, Parcel &parcel) return E_OK; } -int SerializeTextValue(const DataValue &dataValue, Parcel &parcel) -{ - std::string val; - (void)dataValue.GetText(val); - return parcel.WriteString(val); -} - -int DeSerializeTextValue(DataValue &dataValue, Parcel &parcel) -{ - std::string val; - (void)parcel.ReadString(val); - if (parcel.IsError()) { - return -E_PARSE_FAIL; - } - dataValue = val; - return E_OK; -} - int SerializeBlobValue(const DataValue &dataValue, Parcel &parcel) { Blob val; @@ -257,12 +234,12 @@ int SerializeBlobValue(const DataValue &dataValue, Parcel &parcel) return parcel.WriteBlob(reinterpret_cast(val.GetData()), size); } -int DeSerializeBlobValue(DataValue &dataValue, Parcel &parcel) +int DeSerializeBlobByType(DataValue &dataValue, Parcel &parcel, StorageType type) { - Blob val; uint32_t blobLength = 0; (void)parcel.ReadUInt32(blobLength); if (blobLength == 0) { + dataValue.ResetValue(); return E_OK; } char array[blobLength]; @@ -270,13 +247,34 @@ int DeSerializeBlobValue(DataValue &dataValue, Parcel &parcel) if (parcel.IsError()) { return -E_PARSE_FAIL; } - int errCode = val.WriteBlob(reinterpret_cast(array), blobLength); - if (errCode == E_OK) { - dataValue = val; + int errCode = -E_NOT_SUPPORT; + if (type == StorageType::STORAGE_TYPE_TEXT) { + errCode = dataValue.SetText(reinterpret_cast(array), blobLength); + } else if (type == StorageType::STORAGE_TYPE_BLOB) { + Blob val; + errCode = val.WriteBlob(reinterpret_cast(array), blobLength); + if (errCode == E_OK) { + errCode = dataValue.SetBlob(val); + } } return errCode; } +int DeSerializeBlobValue(DataValue &dataValue, Parcel &parcel) +{ + return DeSerializeBlobByType(dataValue, parcel, StorageType::STORAGE_TYPE_BLOB); +} + +int SerializeTextValue(const DataValue &dataValue, Parcel &parcel) +{ + return SerializeBlobValue(dataValue, parcel); +} + +int DeSerializeTextValue(DataValue &dataValue, Parcel &parcel) +{ + return DeSerializeBlobByType(dataValue, parcel, StorageType::STORAGE_TYPE_TEXT); +} + struct FunctionEntry { std::function serializeFunc; std::function deSerializeFunc; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp index 66c4735c4..f558c1781 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -86,12 +86,13 @@ uint64_t SyncAbleEngine::GetTimeStamp() return syncer_.GetTimeStamp(); } -int SyncAbleEngine::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +int SyncAbleEngine::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, + const std::string &tableName) { if (!started_) { StartSyncer(); } - return syncer_.EraseDeviceWaterMark(deviceId, isNeedHash); + return syncer_.EraseDeviceWaterMark(deviceId, isNeedHash, tableName); } // Start syncer diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h index 08e212030..e43d5194b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h @@ -46,7 +46,7 @@ public: // Get The current virtual timestamp uint64_t GetTimeStamp(); - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash); + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, const std::string &tableName = ""); int GetLocalIdentity(std::string &outTarget); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h index fe177b5d3..d42fc8347 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h @@ -75,7 +75,8 @@ public: virtual void EnableAutoSync(bool enable) = 0; // delete specified device's watermark - virtual int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) = 0; + virtual int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, + const std::string &tableName = "") = 0; // Local data changed callback virtual void LocalDataChanged(int notifyEvent) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h index b2b698e6e..52be0f72a 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h @@ -60,7 +60,8 @@ public: void EnableAutoSync(bool enable) override; // delete specified device's watermark - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, + const std::string &tableName = "") override; // Local data changed callback void LocalDataChanged(int notifyEvent) override; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp index 41dd2e0cd..c60415b26 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp @@ -145,14 +145,14 @@ TimeOffset Metadata::GetLocalTimeOffset() const return localTimeOffset; } -int Metadata::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +int Metadata::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, const std::string &tableName) { // try to erase all the waterMark // erase deleteSync recv waterMark WaterMark waterMark = 0; int errCodeDeleteSync = SetRecvDeleteSyncWaterMark(deviceId, waterMark); // erase querySync recv waterMark - int errCodeQuerySync = ResetRecvQueryWaterMark(deviceId); + int errCodeQuerySync = ResetRecvQueryWaterMark(deviceId, tableName); // peerWaterMark must be erased at last int errCode = SavePeerWaterMark(deviceId, 0, isNeedHash); if (errCode != E_OK) { @@ -452,9 +452,9 @@ int Metadata::SetRecvDeleteSyncWaterMark(const DeviceID &deviceId, const WaterMa return querySyncWaterMarkHelper_.SetRecvDeleteSyncWaterMark(deviceId, waterMark); } -int Metadata::ResetRecvQueryWaterMark(const DeviceID &deviceId) +int Metadata::ResetRecvQueryWaterMark(const DeviceID &deviceId, const std::string &tableName) { - return querySyncWaterMarkHelper_.ResetRecvQueryWaterMark(deviceId); + return querySyncWaterMarkHelper_.ResetRecvQueryWaterMark(deviceId, tableName); } void Metadata::GetDbCreateTime(const DeviceID &deviceId, uint64_t &outValue) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.h b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.h index c0d280b12..0cbcbfa96 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.h @@ -58,7 +58,7 @@ public: TimeOffset GetLocalTimeOffset() const; - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash); + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, const std::string &tableName = ""); void SetLastLocalTime(TimeStamp lastLocalTime); @@ -141,7 +141,7 @@ private: int LoadDeviceIdDataToMap(const Key &key); // reset the waterMark to zero - int ResetRecvQueryWaterMark(const DeviceID &deviceId); + int ResetRecvQueryWaterMark(const DeviceID &deviceId, const std::string &tableName = ""); // store localTimeOffset in ram; if change, should add a lock first, change here and metadata, // then release lock diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp index 669dff843..c75695a44 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp @@ -58,7 +58,8 @@ void MultiVerSyncer::EnableAutoSync(bool enable) } } -int MultiVerSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +int MultiVerSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, + const std::string &tableName) { return -E_NOT_SUPPORT; } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.h index 97b9c6b02..9c93d4d9d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.h @@ -30,7 +30,8 @@ public: void EnableAutoSync(bool enable) override; // delete specified device's watermark - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, + const std::string &tableName = "") override; // Local data changed callback void LocalDataChanged(int notifyEvent) override; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp index 7227596e0..307682bc3 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp @@ -547,12 +547,16 @@ int QuerySyncWaterMarkHelper::RemoveLeastUsedQuerySyncItems(const std::vector autoLock(queryWaterMarkLock_); std::string prefixKeyStr = QUERY_SYNC_PREFIX_KEY + DBCommon::TransferHashString(deviceId); + if (!prefixKeyStr.empty()) { + prefixKeyStr += tableName; + } + // remove in db Key prefixKey; DBCommon::StringToVector(prefixKeyStr, prefixKey); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.h b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.h index e9f82e9a5..6491633cc 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.h @@ -90,7 +90,7 @@ public: int RemoveLeastUsedQuerySyncItems(const std::vector &querySyncIds); // reset the waterMark to zero - int ResetRecvQueryWaterMark(const DeviceID &deviceId); + int ResetRecvQueryWaterMark(const DeviceID &deviceId, const std::string &tableName = ""); static std::string GetQuerySyncPrefixKey(); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index 05117ece3..214095e88 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -329,7 +329,7 @@ int SingleVerDataSync::GetUnsyncData(SingleVerSyncTaskContext *context, std::vec int errCode; WaterMark startMark = 0; SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; - GetLocalWaterMark(curType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), context, startMark); + GetLocalWaterMark(curType, context->GetQuerySyncId(), context, startMark); WaterMark endMark = MAX_TIMESTAMP; if ((endMark == 0) || (startMark > endMark)) { return E_OK; @@ -435,7 +435,7 @@ SyncTimeRange SingleVerDataSync::GetSyncDataTimeRange(SyncType syncType, SingleV { WaterMark localMark = 0; WaterMark deleteMark = 0; - GetLocalWaterMark(syncType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), context, localMark); + GetLocalWaterMark(syncType, context->GetQuerySyncId(), context, localMark); GetLocalDeleteSyncWaterMark(context, deleteMark); if (syncType != SyncType::QUERY_SYNC_TYPE) { return GetFullSyncDataTimeRange(inData, localMark, isUpdate); @@ -538,7 +538,7 @@ int SingleVerDataSync::SaveLocalWaterMark(SyncType syncType, const SingleVerSync WaterMark localMark = 0; int errCode = E_OK; const std::string &deviceId = context->GetDeviceId(); - std::string queryId = GetQuerySyncId(context, context->GetQuery().GetIdentify()); + std::string queryId = context->GetQuerySyncId(); if (syncType != SyncType::QUERY_SYNC_TYPE) { if (isCheckBeforUpdate) { GetLocalWaterMark(syncType, queryId, context, localMark); @@ -572,7 +572,7 @@ int SingleVerDataSync::SaveLocalWaterMark(SyncType syncType, const SingleVerSync if (isNeedUpdateDeleteMark) { LOGD("label=%s,dev=%s,deleteEndTime=%llu", label_.c_str(), STR_MASK(GetDeviceId()), dataTimeRange.deleteEndTime); - errCode = metadata_->SetSendDeleteSyncWaterMark(GetDeleteSyncId(context), dataTimeRange.deleteEndTime); + errCode = metadata_->SetSendDeleteSyncWaterMark(context->GetDeleteSyncId(), dataTimeRange.deleteEndTime); } } if (errCode != E_OK) { @@ -599,7 +599,7 @@ void SingleVerDataSync::GetPeerDeleteSyncWaterMark(const DeviceID &deviceId, Wat void SingleVerDataSync::GetLocalDeleteSyncWaterMark(const SingleVerSyncTaskContext *context, WaterMark &waterMark) const { - metadata_->GetSendDeleteSyncWaterMark(GetDeleteSyncId(context), waterMark, context->IsAutoLiftWaterMark()); + metadata_->GetSendDeleteSyncWaterMark(context->GetDeleteSyncId(), waterMark, context->IsAutoLiftWaterMark()); } void SingleVerDataSync::GetLocalWaterMark(SyncType syncType, const std::string &queryIdentify, @@ -630,7 +630,7 @@ int SingleVerDataSync::RemoveDeviceDataHandle(SingleVerSyncTaskContext *context, SyncType curType = SyncOperation::GetSyncType(packet->GetMode()); WaterMark packetLocalMark = packet->GetLocalWaterMark(); WaterMark peerMark = 0; - GetPeerWaterMark(curType, GetQuerySyncId(context, packet->GetQueryId()), context->GetDeviceId(), peerMark); + GetPeerWaterMark(curType, context->GetQuerySyncId(), context->GetDeviceId(), peerMark); isNeedClearRemoteData = ((packetLocalMark == 0) && (peerMark != 0)); } if (!isNeedClearRemoteData) { @@ -674,7 +674,7 @@ int SingleVerDataSync::DealRemoveDeviceDataByAck(SingleVerSyncTaskContext *conte isNeedClearRemoteData = ((localMark != 0) && (ackWaterMark == 0)); } else { WaterMark peerMark = 0; - GetPeerWaterMark(curType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), + GetPeerWaterMark(curType, context->GetQuerySyncId(), context->GetDeviceId(), peerMark); isNeedClearRemoteData = ((reserved[ACK_PACKET_RESERVED_INDEX_LOCAL_WATER_MARK] == 0) && (peerMark != 0)); } @@ -780,7 +780,7 @@ void SingleVerDataSync::FillDataRequestPacket(DataRequestPacket *packet, SingleV bool needCompressOnSync = false; uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); - std::string id = GetQuerySyncId(context, context->GetQuery().GetIdentify()); + std::string id = context->GetQuerySyncId(); GetLocalWaterMark(curType, id, context, localMark); GetPeerWaterMark(curType, id, context->GetDeviceId(), peerMark); GetLocalDeleteSyncWaterMark(context, deleteMark); @@ -801,7 +801,7 @@ void SingleVerDataSync::FillDataRequestPacket(DataRequestPacket *packet, SingleV packet->SetSessionId(context->GetRequestSessionId()); } packet->SetQuery(context->GetQuery()); - packet->SetQueryId(GetQuerySyncId(context, context->GetQuery().GetIdentify())); + packet->SetQueryId(context->GetQuerySyncId()); CompressAlgorithm curAlgo = context->ChooseCompressAlgo(); if (needCompressOnSync && curAlgo != CompressAlgorithm::NONE) { packet->SetCompressDataMark(); @@ -918,9 +918,9 @@ int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) WaterMark peerMark = 0; WaterMark localMark = 0; WaterMark deleteMark = 0; - GetPeerWaterMark(syncType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), + GetPeerWaterMark(syncType, context->GetQuerySyncId(), context->GetDeviceId(), peerMark); - GetLocalWaterMark(syncType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), context, localMark); + GetLocalWaterMark(syncType, context->GetQuerySyncId(), context, localMark); GetLocalDeleteSyncWaterMark(context, deleteMark); uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); WaterMark endMark = context->GetEndMark(); @@ -931,7 +931,7 @@ int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) packet->SetEndWaterMark(endMark); packet->SetSessionId(context->GetRequestSessionId()); packet->SetQuery(context->GetQuery()); - packet->SetQueryId(GetQuerySyncId(context, context->GetQuery().GetIdentify())); + packet->SetQueryId(context->GetQuerySyncId()); packet->SetLastSequence(); SetPacketId(packet, context, version); @@ -1019,7 +1019,7 @@ void SingleVerDataSync::UpdatePeerWaterMark(SyncType syncType, const std::string if (peerDeletedWatermark != 0) { LOGD("label=%s,dev=%s,peerDeletedTime=%llu", label_.c_str(), STR_MASK(GetDeviceId()), peerDeletedWatermark); - errCode = metadata_->SetRecvDeleteSyncWaterMark(GetDeleteSyncId(context), peerDeletedWatermark); + errCode = metadata_->SetRecvDeleteSyncWaterMark(context->GetDeleteSyncId(), peerDeletedWatermark); } } if (errCode != E_OK) { @@ -1650,7 +1650,7 @@ bool SingleVerDataSync::WaterMarkErrHandle(SyncType syncType, SingleVerSyncTaskC WaterMark deletedMark = 0; GetPeerWaterMark(syncType, packet->GetQueryId(), context->GetDeviceId(), peerMark); if (syncType == SyncType::QUERY_SYNC_TYPE) { - GetPeerDeleteSyncWaterMark(GetDeleteSyncId(context), deletedMark); + GetPeerDeleteSyncWaterMark(context->GetDeleteSyncId(), deletedMark); } if (syncType != SyncType::QUERY_SYNC_TYPE && packetLocalMark > peerMark) { LOGI("[DataSync][DataRequestRecv] packetLocalMark=%llu,current=%llu", packetLocalMark, peerMark); @@ -1769,7 +1769,7 @@ void SingleVerDataSync::SetAckPacket(DataAckPacket &ackPacket, SingleVerSyncTask // while recv is not E_OK, data is peerMark, reserve[2] is deletedPeerMark value if (curType == SyncType::QUERY_SYNC_TYPE && recvCode != WATER_MARK_INVALID) { WaterMark deletedPeerMark; - GetPeerDeleteSyncWaterMark(GetDeleteSyncId(context), deletedPeerMark); + GetPeerDeleteSyncWaterMark(context->GetDeleteSyncId(), deletedPeerMark); reserved.push_back(deletedPeerMark); // query sync mode, reserve[2] store deletedPeerMark value } ackPacket.SetReserved(reserved); @@ -1884,7 +1884,7 @@ void SingleVerDataSync::FillRequestReSendPacket(const SingleVerSyncTaskContext * { SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; WaterMark peerMark = 0; - GetPeerWaterMark(curType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), context->GetDeviceId(), + GetPeerWaterMark(curType, context->GetQuerySyncId(), context->GetDeviceId(), peerMark); uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); // transer reSend mode, RESPONSE_PULL transfer to push or query push @@ -1907,7 +1907,7 @@ void SingleVerDataSync::FillRequestReSendPacket(const SingleVerSyncTaskContext * packet->SetEndWaterMark(context->GetEndMark()); packet->SetQuery(context->GetQuery()); } - packet->SetQueryId(GetQuerySyncId(context, context->GetQuery().GetIdentify())); + packet->SetQueryId(context->GetQuerySyncId()); if (version > SOFTWARE_VERSION_RELEASE_2_0) { std::vector reserved {reSendInfo.packetId}; packet->SetReserved(reserved); @@ -2355,30 +2355,6 @@ void SingleVerDataSync::ControlAckErrorHandle(const SingleVerSyncTaskContext *co } } -std::string SingleVerDataSync::GetQuerySyncId(const SingleVerSyncTaskContext *context, const std::string &queryId) const -{ - std::string id; -#ifdef RELATIONAL_STORE - if (storage_->GetInterfaceType() == ISyncInterface::SYNC_RELATION) { - id += context->GetQuery().GetRelationTableName(); - } -#endif - id += queryId; - return id; -} - -std::string SingleVerDataSync::GetDeleteSyncId(const SingleVerSyncTaskContext *context) const -{ - std::string id; - id += context->GetDeviceId(); -#ifdef RELATIONAL_STORE - if (storage_->GetInterfaceType() == ISyncInterface::SYNC_RELATION) { - id += context->GetQuery().GetRelationTableName(); - } -#endif - return id; -} - void SingleVerDataSync::PutDataMsg(Message *message) { return msgSchedule_.PutMsg(message); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h index 8788197e7..ea65bf4da 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h @@ -297,9 +297,6 @@ protected: int SendControlAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, uint32_t controlCmdType, const CommErrHandler &handler = nullptr); - std::string GetQuerySyncId(const SingleVerSyncTaskContext *context, const std::string &queryId) const; - std::string GetDeleteSyncId(const SingleVerSyncTaskContext *context) const; - uint32_t mtuSize_; SyncGenericInterface* storage_; ICommunicator* communicateHandle_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.cpp new file mode 100644 index 000000000..842fb0b84 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "single_ver_kv_sync_task_context.h" + +namespace DistributedDB { +SingleVerKvSyncTaskContext::SingleVerKvSyncTaskContext() + : SingleVerSyncTaskContext() +{} + +SingleVerKvSyncTaskContext::~SingleVerKvSyncTaskContext() +{ +} + +std::string SingleVerKvSyncTaskContext::GetQuerySyncId() const +{ + return query_.GetIdentify(); +} + +std::string SingleVerKvSyncTaskContext::GetDeleteSyncId() const +{ + return GetDeviceId(); +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.h new file mode 100644 index 000000000..b30da0f23 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SINGLE_VER_KV_SYNC_TASK_CONTEXT_H +#define SINGLE_VER_KV_SYNC_TASK_CONTEXT_H + +#include "single_ver_sync_task_context.h" + +namespace DistributedDB { +class SingleVerKvSyncTaskContext : public SingleVerSyncTaskContext { +public: + + explicit SingleVerKvSyncTaskContext(); + + DISABLE_COPY_ASSIGN_MOVE(SingleVerKvSyncTaskContext); + + std::string GetQuerySyncId() const override; + std::string GetDeleteSyncId() const override; + +protected: + ~SingleVerKvSyncTaskContext() override; +}; +} +#endif // SINGLE_VER_KV_SYNC_TASK_CONTEXT_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp index f3efa44d7..49ae80f43 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp @@ -66,14 +66,6 @@ void SingleVerKVSyncer::EnableAutoSync(bool enable) } } -int SingleVerKVSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) -{ - if (metadata_ == nullptr) { - return -E_NOT_INIT; - } - return metadata_->EraseDeviceWaterMark(deviceId, isNeedHash); -} - // Local data changed callback void SingleVerKVSyncer::LocalDataChanged(int notifyEvent) { @@ -129,8 +121,7 @@ void SingleVerKVSyncer::RemoteDataChanged(const std::string &device) std::string appId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); std::string storeId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); RuntimeContext::GetInstance()->NotifyDatabaseStatusChange(userId, appId, storeId, device, true); - // while remote db is online again, need to do abilitySync - static_cast(syncEngine_)->SetIsNeedResetAbilitySync(device, true); + SingleVerSyncer::RemoteDataChanged(device); if (autoSyncEnable_) { RefObject::IncObjRef(syncEngine_); int retCode = RuntimeContext::GetInstance()->ScheduleTask([this, device] { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.h index cb02a472c..dc601ff0a 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.h @@ -27,9 +27,6 @@ public: // Enable auto sync function void EnableAutoSync(bool enable) override; - // delete specified device's watermark - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; - // Local data changed callback void LocalDataChanged(int notifyEvent) override; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.cpp new file mode 100644 index 000000000..907db1520 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "single_ver_relational_sync_task_context.h" +#include "db_common.h" + +#ifdef RELATIONAL_STORE +namespace DistributedDB { +SingleVerRelationalSyncTaskContext::SingleVerRelationalSyncTaskContext() + : SingleVerSyncTaskContext() +{} + +SingleVerRelationalSyncTaskContext::~SingleVerRelationalSyncTaskContext() +{ +} + +std::string SingleVerRelationalSyncTaskContext::GetQuerySyncId() const +{ + return querySyncId_; +} + +std::string SingleVerRelationalSyncTaskContext::GetDeleteSyncId() const +{ + return deleteSyncId_; +} + +void SingleVerRelationalSyncTaskContext::Clear() +{ + querySyncId_.clear(); + deleteSyncId_.clear(); + SingleVerSyncTaskContext::Clear(); +} + +void SingleVerRelationalSyncTaskContext::CopyTargetData(const ISyncTarget *target, const TaskParam &TaskParam) +{ + SingleVerSyncTaskContext::CopyTargetData(target, TaskParam); + querySyncId_ = query_.GetRelationTableName() + query_.GetIdentify(); // save as deviceId + tableName + queryId + deleteSyncId_ = GetDeviceId() + query_.GetRelationTableName(); // save as deviceId + tableName +} +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.h new file mode 100644 index 000000000..b5f0d338d --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SINGLE_VER_RELATIONAL_SYNC_TASK_CONTEXT_H +#define SINGLE_VER_RELATIONAL_SYNC_TASK_CONTEXT_H + +#ifdef RELATIONAL_STORE +#include "single_ver_sync_task_context.h" + +namespace DistributedDB { +class SingleVerRelationalSyncTaskContext : public SingleVerSyncTaskContext { +public: + + explicit SingleVerRelationalSyncTaskContext(); + + DISABLE_COPY_ASSIGN_MOVE(SingleVerRelationalSyncTaskContext); + + void Clear() override; + std::string GetQuerySyncId() const override; + std::string GetDeleteSyncId() const override; + +protected: + ~SingleVerRelationalSyncTaskContext() override; + void CopyTargetData(const ISyncTarget *target, const TaskParam &taskParam) override; + + std::string querySyncId_; + std::string deleteSyncId_; +}; +} +#endif // RELATIONAL_STORE +#endif // SINGLE_VER_RELATIONAL_SYNC_TASK_CONTEXT_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp index 7cb09883f..7e951f458 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp @@ -137,17 +137,25 @@ void SingleVerRelationalSyncer::EnableAutoSync(bool enable) { } -int SingleVerRelationalSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) -{ - return E_OK; -} - void SingleVerRelationalSyncer::LocalDataChanged(int notifyEvent) { } -void SingleVerRelationalSyncer::RemoteDataChanged(const std::string &device) +int SingleVerRelationalSyncer::SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, + const std::vector &devices) const { + if (!isQuerySync) { + return E_OK; + } + int errCode = static_cast(syncInterface_)->CheckAndInitQueryCondition(query); + if (errCode != E_OK) { + LOGE("[SingleVerRelationalSyncer] QuerySyncObject check failed"); + return errCode; + } + if (mode == SUBSCRIBE_QUERY) { + return -E_NOT_SUPPORT; + } + return E_OK; } } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h index 15e4ef814..f4696e2c7 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h @@ -27,15 +27,14 @@ public: void EnableAutoSync(bool enable) override; - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; - void LocalDataChanged(int notifyEvent) override; protected: int PrepareSync(const SyncParma ¶m, uint32_t syncId) override; - void RemoteDataChanged(const std::string &device) override; + int SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, + const std::vector &devices) const override; private: diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp index bec6bd4bf..9376ce40d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp @@ -16,12 +16,27 @@ #include "single_ver_sync_engine.h" #include "db_common.h" #include "single_ver_sync_task_context.h" +#include "single_ver_kv_sync_task_context.h" +#include "single_ver_relational_sync_task_context.h" #include "log_print.h" namespace DistributedDB { ISyncTaskContext *SingleVerSyncEngine::CreateSyncTaskContext() { - auto context = new (std::nothrow) SingleVerSyncTaskContext(); + SingleVerSyncTaskContext *context = nullptr; + switch (syncInterface_->GetInterfaceType()) { + case ISyncInterface::SYNC_SVD: + context = new (std::nothrow) SingleVerKvSyncTaskContext(); + break; +#ifdef RELATIONAL_STORE + case ISyncInterface::SYNC_RELATION: + context = new (std::nothrow) SingleVerRelationalSyncTaskContext(); + break; +#endif + default: + break; + } + if (context == nullptr) { LOGE("[SingleVerSyncEngine][CreateSyncTaskContext] create failed, may be out of memory"); return nullptr; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h index 85f29c212..754896de9 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h @@ -23,6 +23,7 @@ #include "db_ability.h" #include "query_sync_object.h" +#include "schema.h" #include "single_ver_kvdb_sync_interface.h" #include "single_ver_sync_target.h" #include "subscribe_manager.h" @@ -136,14 +137,19 @@ public: void SaveLastPushTaskExecStatus(int finalStatus) override; void ResetLastPushTaskStatus() override; + + virtual std::string GetQuerySyncId() const = 0; + virtual std::string GetDeleteSyncId() const = 0; protected: ~SingleVerSyncTaskContext() override; void CopyTargetData(const ISyncTarget *target, const TaskParam &taskParam) override; + // For querySync + QuerySyncObject query_; + bool isQuerySync_ = false; private: int GetCorrectedSendWaterMarkForCurrentTask(uint64_t &waterMark) const; -private: constexpr static int64_t REDUNDACE_WATER_MARK = 1 * 1000LL * 1000LL * 10LL; // 1s DECLARE_OBJECT_TAG(SingleVerSyncTaskContext); @@ -162,10 +168,6 @@ private: // is receive waterMark err, peerWaterMark bigger than remote localWaterMark bool isReceiveWaterMarkErr_ = false; - // For querySync - QuerySyncObject query_; - bool isQuerySync_ = false; - // For db ability DbAbility remoteDbAbility_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp index af13a3a95..3069f5e90 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp @@ -18,6 +18,13 @@ #include "single_ver_sync_engine.h" namespace DistributedDB { +void SingleVerSyncer::RemoteDataChanged(const std::string &device) +{ + LOGI("[SingleVerSyncer] device online dev %s", STR_MASK(device)); + // while remote db is online again, need to do abilitySync + static_cast(syncEngine_)->SetIsNeedResetAbilitySync(device, true); +} + void SingleVerSyncer::RemoteDeviceOffline(const std::string &device) { LOGI("[SingleVerRelationalSyncer] device offline dev %s", STR_MASK(device)); @@ -30,6 +37,15 @@ void SingleVerSyncer::RemoteDeviceOffline(const std::string &device) RefObject::DecObjRef(syncEngine_); } +int SingleVerSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, + const std::string &tableName) +{ + if (metadata_ == nullptr) { + return -E_NOT_INIT; + } + return metadata_->EraseDeviceWaterMark(deviceId, isNeedHash, tableName); +} + int SingleVerSyncer::SetStaleDataWipePolicy(WipePolicy policy) { std::lock_guard lock(syncerLock_); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.h index b88b6c520..dab50ddb8 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.h @@ -20,11 +20,17 @@ namespace DistributedDB { class SingleVerSyncer : public GenericSyncer { public: + void RemoteDataChanged(const std::string &device) override; + void RemoteDeviceOffline(const std::string &device) override; // Set stale data wipe policy int SetStaleDataWipePolicy(WipePolicy policy) override; + // delete specified device's watermark + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, + const std::string &tableName = "") override; + protected: // Create a sync engine, if has memory error, will return nullptr. ISyncEngine *CreateSyncEngine() override; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h index 7456da44c..40e36e811 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h @@ -116,6 +116,7 @@ protected: void GetQueryAutoSyncParam(const std::string &device, const QuerySyncObject &query, InternalSyncParma &outParam); void GetSubscribeSyncParam(const std::string &device, const QuerySyncObject &query, InternalSyncParma &outParam); + ISyncInterface *syncInterface_; // Used to store all send sync task infos (such as pull sync response, and push sync request) std::map syncTaskContextMap_; std::mutex contextMapLock_; @@ -185,7 +186,6 @@ private: static uint8_t GetPermissionCheckFlag(bool isAutoSync, int syncMode); - ISyncInterface *syncInterface_; ICommunicator *communicator_; DeviceManager *deviceManager_; std::function onRemoteDataChanged_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp index 70477fee5..1a8afd21b 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp @@ -270,7 +270,6 @@ std::string SyncOperation::GetQueryId() const return query_.GetIdentify(); } - const std::map &SyncOperation::DBStatusTransMap() { static const std::map transMap = { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp index 3679cd30c..a2a9a20b7 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp @@ -104,12 +104,14 @@ void SyncerProxy::EnableAutoSync(bool enable) syncer_->EnableAutoSync(enable); } -int SyncerProxy::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +int SyncerProxy::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, + const std::string &tableName) { if (syncer_ == nullptr) { - return SyncerFactory::GetSyncer(ISyncInterface::SYNC_SVD)->EraseDeviceWaterMark(deviceId, isNeedHash); + LOGE("[SyncerProxy] Syncer no init, unknown rule to erase waterMark!"); + return -E_NOT_INIT; } - return syncer_->EraseDeviceWaterMark(deviceId, isNeedHash); + return syncer_->EraseDeviceWaterMark(deviceId, isNeedHash, tableName); } void SyncerProxy::LocalDataChanged(int notifyEvent) diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 337180528..d3dd9f341 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -226,6 +226,8 @@ ohos_source_set("src_file") { "../syncer/src/single_ver_sync_state_machine.cpp", "../syncer/src/single_ver_sync_target.cpp", "../syncer/src/single_ver_sync_task_context.cpp", + "../syncer/src/single_ver_kv_sync_task_context.cpp", + "../syncer/src/single_ver_relational_sync_task_context.cpp", "../syncer/src/single_ver_syncer.cpp", "../syncer/src/subscribe_manager.cpp", "../syncer/src/sync_engine.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp index d1c463efd..b1a4da85c 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp @@ -17,7 +17,7 @@ #include "ability_sync.h" #include "distributeddb_tools_unit_test.h" -#include "single_ver_sync_task_context.h" +#include "single_ver_kv_sync_task_context.h" #include "sync_types.h" #include "version.h" #include "virtual_communicator_aggregator.h" @@ -434,7 +434,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestReceiveTest001, TestSize.Level0) */ Message msg1(ABILITY_SYNC_MESSAGE); msg1.SetMessageType(TYPE_REQUEST); - SingleVerSyncTaskContext *context = new (std::nothrow) SingleVerSyncTaskContext(); + SingleVerSyncTaskContext *context = new (std::nothrow) SingleVerKvSyncTaskContext(); ASSERT_TRUE(context != nullptr); EXPECT_EQ(async.RequestRecv(nullptr, context), -E_INVALID_ARGS); EXPECT_EQ(async.RequestRecv(&msg1, nullptr), -E_INVALID_ARGS); @@ -503,7 +503,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, AckReceiveTest001, TestSize.Level0) * @tc.steps: step2. call AckRecv, set inMsg nullptr or set context nullptr * @tc.expected: step2. AckRecv return -E_INVALID_ARGS */ - SingleVerSyncTaskContext *context = new (std::nothrow) SingleVerSyncTaskContext(); + SingleVerSyncTaskContext *context = new (std::nothrow) SingleVerKvSyncTaskContext(); ASSERT_TRUE(context != nullptr); Message msg1(ABILITY_SYNC_MESSAGE); msg1.SetMessageType(TYPE_RESPONSE); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp index c5c0afa8d..fabb61c5f 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp @@ -18,6 +18,7 @@ #include "distributeddb_tools_unit_test.h" #include "single_ver_data_message_schedule.h" #include "single_ver_data_packet.h" +#include "single_ver_kv_sync_task_context.h" using namespace testing::ext; using namespace DistributedDB; @@ -65,7 +66,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule001, TestSize.Level0) * @tc.expected: put msg ok */ SingleVerDataMessageSchedule msgSchedule; - auto *context = new SingleVerSyncTaskContext(); + auto *context = new SingleVerKvSyncTaskContext(); context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); DataSyncMessageInfo info; info.sessionId_ = 10; @@ -115,7 +116,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule002, TestSize.Level0) * @tc.expected: put msg ok */ SingleVerDataMessageSchedule msgSchedule; - auto *context = new SingleVerSyncTaskContext(); + auto *context = new SingleVerKvSyncTaskContext(); context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_RELEASE_2_0); DataSyncMessageInfo info; bool isNeedHandle = true; @@ -158,7 +159,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule003, TestSize.Level0) * @tc.expected: handle ok */ SingleVerDataMessageSchedule msgSchedule; - auto *context = new SingleVerSyncTaskContext(); + auto *context = new SingleVerKvSyncTaskContext(); context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); DataSyncMessageInfo info; bool isNeedHandle = true; @@ -225,7 +226,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule004, TestSize.Level0) * @tc.expected: put msg ok */ SingleVerDataMessageSchedule msgSchedule; - auto *context = new SingleVerSyncTaskContext(); + auto *context = new SingleVerKvSyncTaskContext(); context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); DataSyncMessageInfo info; info.sessionId_ = 10; @@ -284,7 +285,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule005, TestSize.Level0) * @tc.expected: put msg ok */ SingleVerDataMessageSchedule msgSchedule; - auto *context = new SingleVerSyncTaskContext(); + auto *context = new SingleVerKvSyncTaskContext(); context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); DataSyncMessageInfo info; info.sessionId_ = 10; @@ -341,7 +342,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule006, TestSize.Level0) * @tc.expected: put msg ok */ SingleVerDataMessageSchedule msgSchedule; - auto *context = new SingleVerSyncTaskContext(); + auto *context = new SingleVerKvSyncTaskContext(); context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); DataSyncMessageInfo info; info.sessionId_ = 10; @@ -407,7 +408,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule007, TestSize.Level0) * @tc.expected: put msg ok and get msg seq1_packet4, seq2_packet5 */ SingleVerDataMessageSchedule msgSchedule; - auto *context = new SingleVerSyncTaskContext(); + auto *context = new SingleVerKvSyncTaskContext(); context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); DataSyncMessageInfo info; info.sessionId_ = 10; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp index f002c34f5..0ab564c2b 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp @@ -19,6 +19,7 @@ #include "distributeddb_tools_unit_test.h" #include "isyncer.h" #include "single_ver_sync_state_machine.h" +#include "single_ver_kv_sync_task_context.h" #include "sync_types.h" #include "virtual_single_ver_sync_db_Interface.h" #include "virtual_time_sync_communicator.h" @@ -88,7 +89,7 @@ void DistributedDBTimeSyncTest::SetUp(void) g_timeSyncB = new (std::nothrow) TimeSync(); ASSERT_TRUE(g_timeSyncB != nullptr); - g_syncTaskContext = new (std::nothrow) SingleVerSyncTaskContext(); + g_syncTaskContext = new (std::nothrow) SingleVerKvSyncTaskContext(); ASSERT_TRUE(g_syncTaskContext != nullptr); } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp index 0154d235e..338152b18 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp @@ -15,7 +15,8 @@ #include "generic_virtual_device.h" #include "multi_ver_sync_task_context.h" -#include "single_ver_sync_task_context.h" +#include "single_ver_kv_sync_task_context.h" +#include "single_ver_relational_sync_task_context.h" namespace DistributedDB { GenericVirtualDevice::GenericVirtualDevice(std::string deviceId) @@ -86,11 +87,11 @@ int GenericVirtualDevice::Initialize(VirtualCommunicatorAggregator *comAggregato return -E_NOT_SUPPORT; } if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_SVD) { - context_ = new (std::nothrow) SingleVerSyncTaskContext; + context_ = new (std::nothrow) SingleVerKvSyncTaskContext; subManager_ = std::make_shared(); static_cast(context_)->SetSubscribeManager(subManager_); } else if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_RELATION) { - context_ = new (std::nothrow) SingleVerSyncTaskContext; + context_ = new (std::nothrow) SingleVerRelationalSyncTaskContext; } else { context_ = new (std::nothrow) MultiVerSyncTaskContext; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp index 3a6678ae8..c1185f423 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp @@ -213,9 +213,9 @@ std::vector VirtualRelationalVerSyncDBInterface::GetIdentifier() const void VirtualRelationalVerSyncDBInterface::GetMaxTimeStamp(TimeStamp &stamp) const { for (const auto &item : syncData_) { - for (const auto &[hashKey, virtualRowData] : item.second) { - if (stamp < virtualRowData.logInfo.timestamp) { - stamp = virtualRowData.logInfo.timestamp; + for (const auto &entry : item.second) { + if (stamp < entry.second.logInfo.timestamp) { + stamp = entry.second.logInfo.timestamp; } } } @@ -292,8 +292,8 @@ int VirtualRelationalVerSyncDBInterface::GetAllSyncData(const std::string &table if (syncData_.find(tableName) == syncData_.end()) { return -E_NOT_FOUND; } - for (const auto &[hashKey, virtualRowData] : syncData_[tableName]) { - data.push_back(virtualRowData); + for (const auto &entry : syncData_[tableName]) { + data.push_back(entry.second); } return E_OK; } -- Gitee From 4a289fc2322f0b9a8e941367a9d6aeedeeb08084 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Thu, 13 Jan 2022 20:17:59 +0800 Subject: [PATCH 16/38] InKeys. Signed-off-by: lidwchn --- .../libs/distributeddb/common/src/query.cpp | 6 + .../common/src/query_expression.cpp | 13 ++ .../libs/distributeddb/include/query.h | 3 + .../distributeddb/include/query_expression.h | 6 + .../storage/src/sqlite/query_object.cpp | 24 ++- .../storage/src/sqlite/query_object.h | 2 + .../storage/src/sqlite/query_sync_object.h | 1 + .../src/sqlite/sqlite_query_helper.cpp | 56 +++++- .../storage/src/sqlite/sqlite_query_helper.h | 6 + ...buteddb_interfaces_data_operation_test.cpp | 160 ++++++++++++++++++ 10 files changed, 273 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/query.cpp b/services/distributeddataservice/libs/distributeddb/common/src/query.cpp index 5204d1921..49cbdea95 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/query.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/query.cpp @@ -60,6 +60,12 @@ Query &Query::SuggestIndex(const std::string &indexName) return *this; } +Query &Query::InKeys(const std::set &keys) +{ + queryExpression_.InKeys(keys); + return *this; +} + Query &Query::OrderBy(const std::string &field, bool isAsc) { queryExpression_.OrderBy(field, isAsc); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp index 90ff60ec3..730a6013c 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp @@ -186,6 +186,13 @@ void QueryExpression::QueryBySuggestIndex(const std::string &indexName) suggestIndex_ = indexName; } +void QueryExpression::InKeys(const std::set &keys) +{ + queryInfo_.emplace_back( + QueryObjNode{QueryObjType::IN_KEYS, std::string(), QueryValueType::VALUE_TYPE_NULL, std::vector()}); + keys_ = keys; +} + const std::list &QueryExpression::GetQueryExpression() { if (!GetErrFlag()) { @@ -222,6 +229,11 @@ std::string QueryExpression::GetSuggestIndex() const return suggestIndex_; } +const std::set &QueryExpression::GetKeys() const +{ + return keys_; +} + void QueryExpression::BeginGroup() { queryInfo_.emplace_back(QueryObjNode{QueryObjType::BEGIN_GROUP, std::string(), @@ -241,6 +253,7 @@ void QueryExpression::Reset() prefixKey_.clear(); prefixKey_.shrink_to_fit(); suggestIndex_.clear(); + keys_.clear(); } void QueryExpression::SetErrFlag(bool flag) diff --git a/services/distributeddataservice/libs/distributeddb/include/query.h b/services/distributeddataservice/libs/distributeddb/include/query.h index 91271f6a3..5480939c3 100755 --- a/services/distributeddataservice/libs/distributeddb/include/query.h +++ b/services/distributeddataservice/libs/distributeddb/include/query.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "query_expression.h" #include "types_export.h" @@ -140,6 +141,8 @@ public: DB_API Query &SuggestIndex(const std::string &indexName); + DB_API Query &InKeys(const std::set &keys); + friend class GetQueryInfo; ~Query() = default; diff --git a/services/distributeddataservice/libs/distributeddb/include/query_expression.h b/services/distributeddataservice/libs/distributeddb/include/query_expression.h index 241500207..a864c6111 100755 --- a/services/distributeddataservice/libs/distributeddb/include/query_expression.h +++ b/services/distributeddataservice/libs/distributeddb/include/query_expression.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "types_export.h" @@ -57,6 +58,7 @@ enum class QueryObjType : uint32_t { LIMIT = 0x0701, ORDERBY, SUGGEST_INDEX = 0x0801, + IN_KEYS = 0x0901, }; struct QueryObjNode { @@ -118,6 +120,9 @@ public: std::string GetSuggestIndex() const; + const std::set &GetKeys() const; + void InKeys(const std::set &keys); + const std::list &GetQueryExpression(); void SetErrFlag(bool flag); @@ -133,6 +138,7 @@ private: std::string suggestIndex_; std::string tableName_; bool isTableNameSpecified_; + std::set keys_; }; // specialize for double diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp index 3fec4df8d..d32477223 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp @@ -36,6 +36,7 @@ QueryObject::QueryObject() hasOrderBy_(false), hasLimit_(false), hasPrefixKey_(false), + hasInKeys_(false), orderByCounts_(0) { } @@ -54,6 +55,8 @@ void QueryObject::GetAttrFromQueryObjNodes() hasOrderBy_ = true; } else if (symbolType == PREFIXKEY_SYMBOL) { hasPrefixKey_ = true; + } else if (symbolType == IN_KEYS_SYMBOL) { + hasInKeys_ = true; } } } @@ -65,6 +68,7 @@ QueryObject::QueryObject(const Query &query) hasOrderBy_(false), hasLimit_(false), hasPrefixKey_(false), + hasInKeys_(false), orderByCounts_(0) { QueryExpression queryExpressions = GetQueryInfo::GetQueryExpression(query); @@ -75,6 +79,7 @@ QueryObject::QueryObject(const Query &query) suggestIndex_ = queryExpressions.GetSuggestIndex(); tableName_ = queryExpressions.GetTableName(); isTableNameSpecified_ = queryExpressions.IsTableNameSpecified(); + keys_ = queryExpressions.GetKeys(); } QueryObject::QueryObject(const std::list &queryObjNodes, const std::vector &prefixKey) @@ -87,6 +92,7 @@ QueryObject::QueryObject(const std::list &queryObjNodes, const std hasOrderBy_(false), hasLimit_(false), hasPrefixKey_(false), + hasInKeys_(false); orderByCounts_(0) { GetAttrFromQueryObjNodes(); @@ -117,7 +123,7 @@ SqliteQueryHelper QueryObject::GetQueryHelper(int &errCode) if (errCode != E_OK) { return SqliteQueryHelper(QueryObjInfo{}); } - QueryObjInfo info {schema_, queryObjNodes_, prefixKey_, suggestIndex_, + QueryObjInfo info {schema_, queryObjNodes_, prefixKey_, suggestIndex_, keys_, orderByCounts_, isValid_, hasOrderBy_, hasLimit_, hasPrefixKey_, tableName_}; return SqliteQueryHelper {info}; // compiler RVO by default, and RVO is generally required after C++17 } @@ -168,6 +174,7 @@ void QueryObject::ClearNodesFlag() hasOrderBy_ = false; hasLimit_ = false; hasPrefixKey_ = false; + hasInKeys_ = false; orderByCounts_ = 0; } @@ -232,6 +239,16 @@ int QueryObject::ParseNode(const std::list::iterator &iter) } } else if (symbolType == SUGGEST_INDEX_SYMBOL) { return CheckSuggestIndexFormat(iter); + } else if (symbolType == IN_KEYS_SYMBOL) { + if (hasInKeys_) { + LOGE("Only filter by keys in once!!"); + return -E_INVALID_ARGS; + } + + if (keys_.size() > DBConstant::MAX_BATCH_SIZE) { + return -E_MAX_LIMITS; + } + hasInKeys_ = true; } return E_OK; } @@ -241,7 +258,7 @@ int QueryObject::CheckLinkerBefore(const std::list::iterator &iter auto preIter = std::prev(iter, 1); SymbolType symbolType = SqliteQueryHelper::GetSymbolType(preIter->operFlag); if (symbolType != COMPARE_SYMBOL && symbolType != RELATIONAL_SYMBOL && symbolType != LOGIC_SYMBOL && - symbolType != RANGE_SYMBOL && symbolType != PREFIXKEY_SYMBOL) { + symbolType != RANGE_SYMBOL && symbolType != PREFIXKEY_SYMBOL && symbolType != IN_KEYS_SYMBOL) { LOGE("Must be a comparison operation before the connective! operFlag = %s", VNAME(preIter->operFlag)); return -E_INVALID_QUERY_FORMAT; } @@ -352,7 +369,8 @@ int QueryObject::CheckLimitFormat(const std::list::iterator &iter) bool QueryObject::IsQueryOnlyByKey() const { return std::none_of(queryObjNodes_.begin(), queryObjNodes_.end(), [&](const QueryObjNode &node) { - return node.operFlag != QueryObjType::LIMIT && node.operFlag != QueryObjType::QUERY_BY_KEY_PREFIX; + return node.operFlag != QueryObjType::LIMIT && node.operFlag != QueryObjType::QUERY_BY_KEY_PREFIX && + node.operFlag != QueryObjType::IN_KEYS; }); } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h index b65ef7025..6f732c209 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h @@ -64,6 +64,7 @@ protected: std::vector prefixKey_; std::string tableName_ = "sync_data"; std::string suggestIndex_; + std::set keys_; bool isValid_ = true; @@ -88,6 +89,7 @@ private: bool hasOrderBy_; bool hasLimit_; bool hasPrefixKey_; + bool hasInKeys_; int orderByCounts_; }; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h index 052b9ea29..35d18750f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h @@ -30,6 +30,7 @@ struct ObjContext { std::vector prefixKey{}; std::string suggestIndex{}; std::list queryObjNodes{}; + std::vector keys{}; }; class QuerySyncObject : public QueryObject { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp index 212b13f30..64e3dee73 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp @@ -123,6 +123,7 @@ SqliteQueryHelper::SqliteQueryHelper(const QueryObjInfo &info) prefixKey_(info.prefixKey_), suggestIndex_(info.suggestIndex_), tableName_(info.tableName_), + keys_(info.keys_), orderByCounts_(info.orderByCounts_), isValid_(info.isValid_), transformed_(false), @@ -147,7 +148,7 @@ bool SqliteQueryHelper::FilterSymbolToAddBracketLink(std::string &querySql, bool querySql += isNeedLink ? " AND (" : " ("; isNeedEndBracket = true; break; - } else if (symbolType == LOGIC_SYMBOL || symbolType == PREFIXKEY_SYMBOL) { + } else if (symbolType == LOGIC_SYMBOL || symbolType == PREFIXKEY_SYMBOL || symbolType == IN_KEYS_SYMBOL) { continue; } else { break; @@ -277,6 +278,7 @@ int SqliteQueryHelper::GetQuerySql(std::string &sql, bool onlyRowid) const std::string &querySqlForUse = (onlyRowid ? PRE_QUERY_ROWID_SQL : PRE_QUERY_KV_SQL); sql = AssembleSqlForSuggestIndex(querySqlForUse, FILTER_NATIVE_DATA_SQL); sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); + sql = keys_.empty() ? sql : (sql + "AND " + MapKeysInToSql(keys_.size())); if (transformed_) { LOGD("This query object has been parsed."); sql += querySql_; @@ -300,6 +302,7 @@ int SqliteQueryHelper::GetSyncDataCheckSql(std::string &sql) } sql = PRE_QUERY_ITEM_SQL + tableName_ + " WHERE hash_key=? AND (flag&0x01=0) "; sql += hasPrefixKey_ ? " AND (key>=? AND key<=?) " : ""; + sql = keys_.empty() ? sql : (sql + "AND " + MapKeysInToSql(keys_.size())); if (!transformed_) { errCode = ToQuerySql(); if (errCode != E_OK) { @@ -354,6 +357,7 @@ int SqliteQueryHelper::GetCountQuerySql(std::string &sql) } sql = AssembleSqlForSuggestIndex(PRE_GET_COUNT_SQL, FILTER_NATIVE_DATA_SQL); sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); + sql = keys_.empty() ? sql : (sql + "AND " + MapKeysInToSql(keys_.size())); sql += countSql_; return E_OK; } @@ -377,6 +381,11 @@ int SqliteQueryHelper::GetQuerySqlStatement(sqlite3 *dbHandle, const std::string index = 3; // begin from 3rd args } + errCode = BindKeysToStmt(keys_, statement, index); + if (errCode != E_OK) { + return errCode; + } + for (const QueryObjNode &objNode : queryObjNodes_) { errCode = BindFieldValue(statement, objNode, index); if (errCode != E_OK) { @@ -412,6 +421,11 @@ int SqliteQueryHelper::GetQuerySqlStatement(sqlite3 *dbHandle, bool onlyRowid, s index = 3; // begin from 3rd args } + errCode = BindKeysToStmt(keys_, statement, index); + if (errCode != E_OK) { + return errCode; + } + for (const QueryObjNode &objNode : queryObjNodes_) { errCode = BindFieldValue(statement, objNode, index); if (errCode != E_OK) { @@ -447,6 +461,11 @@ int SqliteQueryHelper::GetCountSqlStatement(sqlite3 *dbHandle, sqlite3_stmt *&co index = 3; // begin from 3rd args } + errCode = BindKeysToStmt(keys_, countStmt, index); + if (errCode != E_OK) { + return errCode; + } + for (const QueryObjNode &objNode : queryObjNodes_) { if (GetSymbolType(objNode.operFlag) == SPECIAL_SYMBOL) { continue; @@ -474,6 +493,7 @@ int SqliteQueryHelper::GetSyncDataQuerySql(std::string &sql, bool hasSubQuery) sql = AssembleSqlForSuggestIndex(PRE_QUERY_ITEM_SQL + tableName_ + " ", FILTER_REMOTE_QUERY); sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); + sql = keys_.empty() ? sql : (sql + "AND " + MapKeysInToSql(keys_.size())); sql = hasSubQuery ? sql : (sql + " AND (timestamp>=? AND timestamp=? @@ -917,4 +942,33 @@ int SqliteQueryHelper::GetRelationalQueryStatement(sqlite3 *dbHandle, uint64_t b } return errCode; } + +std::string SqliteQueryHelper::MapKeysInToSql(size_t keysNum) const +{ + std::string resultSql = "key IN ("; + for (size_t i = 0; i < keysNum; i++) { + if (i != 0) { + resultSql += ", "; + } + resultSql += "? "; + } + resultSql += ") "; + return resultSql; +} + +int SqliteQueryHelper::BindKeysToStmt(std::set &keys, sqlite3_stmt *&statement, int &index) const +{ + if (!keys_.empty()) { + int errCode = E_OK; + for (const auto &key : keys) { + errCode = SQLiteUtils::BindBlobToStatement(statement, index, key); + if (errCode != E_OK) { + LOGE("[Query] Get statement when bind keys failed, errCode = %d", errCode); + return errCode; + } + index++; + } + } + return E_OK; +} } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.h index fc3188885..b8fa259eb 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.h @@ -16,6 +16,7 @@ #define SQLITE_QUERY_HELPER_H #include +#include #include #include @@ -32,6 +33,7 @@ struct QueryObjInfo { std::list queryObjNodes_; std::vector prefixKey_; std::string suggestIndex_; + std::set keys_; int orderByCounts_ = 0; // Record processing to which orderBy node bool isValid_ = true; bool hasOrderBy_ = false; @@ -50,6 +52,7 @@ enum SymbolType : uint32_t { LINK_SYMBOL = 0x0600, // use to link relatonal symbol SPECIAL_SYMBOL = 0x0700, // need special precess and need at the last SUGGEST_INDEX_SYMBOL = 0x0800, + IN_KEYS_SYMBOL = 0x0900, }; class SqliteQueryHelper final { @@ -112,12 +115,15 @@ private: int BindTimeRange(sqlite3_stmt *&statement, int &index, uint64_t beginTime, uint64_t endTime) const; int BindObjNodes(sqlite3_stmt *&statement, int &index) const; int GetSubscribeCondition(const std::string &accessStr, std::string &conditionStr); + std::string MapKeysInToSql(size_t keysNum) const; + int BindKeysToStmt(std::set &keys, sqlite3_stmt *&countStmt, int &index) const; SchemaObject schema_; std::list queryObjNodes_; std::vector prefixKey_; std::string suggestIndex_; std::string tableName_; + std::set keys_; std::string querySql_; std::string countSql_; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp index 5956d322b..aef048811 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp @@ -1786,4 +1786,164 @@ HWTEST_F(DistributedDBInterfacesDataOperationTest, PrefixAndOther001, TestSize.L EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtrForQuery), OK); EXPECT_TRUE(g_mgr.DeleteKvStore("PrefixAndOther001") == OK); } + +/** + * @tc.name: InKeys001 + * @tc.desc: InKeys query base function + * @tc.type: FUNC + * @tc.require: AR000EPARK + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeys001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Create a database And Preset Data + */ + KvStoreNbDelegate::Option option = {true, false, false}; + g_mgr.GetKvStore("InKeys001", option, g_kvNbDelegateCallbackForQuery); + ASSERT_NE(g_kvNbDelegatePtrForQuery, nullptr); + EXPECT_EQ(g_kvDelegateStatusForQuery, OK); + + Key key = {'1'}; + DBStatus status = g_kvDelegateCallbackForQuery->Put(key, VALUE_1); + ASSERT_EQ(status, OK); + const int dataSize = 10; + std::set keys; + for (uint8_t i = 0; i < dataSize; i++) { + key.push_back(i); + DBStatus status = g_kvNbDelegatePtrForQuery->Put(key, VALUE_1); + ASSERT_EQ(status, OK); + keys.emplace(key); + key.pop_back(); + } + + /** + * @tc.steps: step2. Call GetEntries With Query, set all keys at Inkeys. + * @tc.expected: step2. Returns KvStoreResultSet, the count is dataSize, + * all data are equals the preset data + */ + KvStoreResultSet *resultSet = nullptr; + g_kvNbDelegatePtrForQuery->GetEntries(Query::Select().InKeys(keys), resultSet); + ASSERT_NE(resultSet, nullptr); + ASSERT_EQ(resultSet->GetCount(), dataSize); + for (int i = 0; i < dataSize; i++) { + resultSet->MoveToPosition(i); + Entry entry; + resultSet->GetEntry(entry); + key.push_back(i); + EXPECT_EQ(key, entry.key); + EXPECT_EQ(entry.value, VALUE_1); + key.pop_back(); + } + g_kvNbDelegatePtrForQuery->CloseResultSet(resultSet); + + /** + * @tc.steps: step3. Call GetEntries With Query, set all keys at Inkeys. + * @tc.expected: step3. Returns KvStoreResultSet, the count is 0, + */ + g_kvNbDelegatePtrForQuery->GetEntries() + g_kvNbDelegatePtrForQuery->GetEntries(Query::Select().InKeys({KEY_7}), resultSet); + ASSERT_NE(resultSet, nullptr); + ASSERT_EQ(resultSet->GetCount(), 0); + g_kvNbDelegatePtrForQuery->CloseResultSet(resultSet); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtrForQuery), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("InKeys001"), OK); +} + +/** + * @tc.name: InKeysLimit001 + * @tc.desc: InKeys query limit verification + * @tc.type: FUNC + * @tc.require: AR000EPARK + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeysLimit001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Create a database + */ + KvStoreNbDelegate::Option option = {true, false, false}; + g_mgr.GetKvStore("InKeysLimit001", option, g_kvNbDelegateCallbackForQuery); + ASSERT_NE(g_kvNbDelegatePtrForQuery, nullptr); + EXPECT_EQ(g_kvDelegateStatusForQuery, OK); + + /** + * @tc.steps: step2. Construct a key set, and the key size over MAX_BATCH_SIZE + */ + std::set keys; + for (uint8_t i = 0; i < DBConstant::MAX_BATCH_SIZE + 1; i++) { + Key key = { i }; + keys.emplace(key); + } + + /** + * @tc.steps: step3. Call GetEntries With Query, set keys at Inkeys. + * @tc.expected: step3. Returns OVER_MAX_LIMITS, the resultSet is nullptr, + */ + KvStoreResultSet *resultSet = nullptr; + DBStatus status = g_kvNbDelegatePtrForQuery->GetEntries(Query::Select().InKeys(keys), resultSet); + EXPECT_EQ(resultSet, nullptr); + + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtrForQuery), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("InKeysLimit001"), OK); +} + +/** + * @tc.name: InKeysAndOthers001 + * @tc.desc: Combination of InKeys query and logical filtering + * @tc.type: FUNC + * @tc.require: AR000EPARK + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeysAndOthers001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Create a database And Preset Data + */ + KvStoreNbDelegate::Option option = {true, false, false}; + option.schema = SCHEMA_DEFINE2; + g_mgr.GetKvStore("InKeysAndOthers001", option, g_kvNbDelegateCallbackForQuery); + ASSERT_NE(g_kvNbDelegatePtrForQuery, nullptr); + EXPECT_EQ(g_kvDelegateStatusForQuery, OK); + + PresetDataForPreifxAndOrderBy001(); + + std::set keys = { KEY_1, KEY_2, KEY_4 }; + std::vector entriesRes; + + /** + * @tc.steps: step2. Call GetEntries With Query, use EqualTo and InKeys + * @tc.expected: step2. Returns OK + */ + Query query1 = Query::Select().EqualTo("$.field_name1", 1).InKeys(keys); + int errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); + EXPECT_EQ(errCode, OK); + + /** + * @tc.steps: step3. Call GetEntries With Query, use InKeys and EqualTo + * @tc.expected: step3. Returns OK + */ + query1 = Query::Select().InKeys(keys).EqualTo("$.field_name1", 1); + int errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); + EXPECT_EQ(errCode, OK); + + /** + * @tc.steps: step4. Call GetEntries With Query, use EqualTo and InKeys and EqualTo, all valid + * @tc.expected: step4. Returns OK + */ + query1 = Query::Select().EqualTo("$.field_name1", 1).InKeys(keys).And().EqualTo("$.field_name2", 2); + int errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); + EXPECT_EQ(errCode, OK); + + /** + * @tc.steps: step4. Call GetEntries With Query, use EqualTo and InKeys and EqualTo, all valid + * @tc.expected: step4. Returns OK + */ + query1 = Query::Select().EqualTo("$.field_name1", 1).InKeys(keys).And().EqualTo("$.field_name1", 2); + int errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); + EXPECT_EQ(errCode, NOT_FOUND); + + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtrForQuery), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("InKeysAndOthers001"), OK); +} #endif \ No newline at end of file -- Gitee From eca23083dd7c5eb5df9ff27a6aab66fd89a32546 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Sat, 15 Jan 2022 14:37:50 +0800 Subject: [PATCH 17/38] InKeys. Query sync and subscribe query. Signed-off-by: lidwchn --- .../storage/src/sqlite/query_object.cpp | 33 ++++- .../storage/src/sqlite/query_object.h | 4 +- .../storage/src/sqlite/query_sync_object.cpp | 97 ++++++++++---- .../storage/src/sqlite/query_sync_object.h | 4 +- .../src/sqlite/sqlite_query_helper.cpp | 33 ++++- .../storage/src/sqlite/sqlite_query_helper.h | 2 + ...buteddb_interfaces_data_operation_test.cpp | 66 ++++++---- .../distributeddb_storage_query_sync_test.cpp | 43 ++++++- ...buteddb_single_ver_p2p_query_sync_test.cpp | 71 +++++++++++ ...eddb_single_ver_p2p_subsribe_sync_test.cpp | 118 +++++++++++++++++- 10 files changed, 407 insertions(+), 64 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp index d32477223..ba23abef6 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp @@ -82,9 +82,11 @@ QueryObject::QueryObject(const Query &query) keys_ = queryExpressions.GetKeys(); } -QueryObject::QueryObject(const std::list &queryObjNodes, const std::vector &prefixKey) +QueryObject::QueryObject(const std::list &queryObjNodes, const std::vector &prefixKey, + const std::set &keys) : queryObjNodes_(queryObjNodes), prefixKey_(prefixKey), + keys_(keys), isValid_(true), initialized_(false), limit_(INVALID_LIMIT), @@ -92,7 +94,7 @@ QueryObject::QueryObject(const std::list &queryObjNodes, const std hasOrderBy_(false), hasLimit_(false), hasPrefixKey_(false), - hasInKeys_(false); + hasInKeys_(false), orderByCounts_(0) { GetAttrFromQueryObjNodes(); @@ -244,11 +246,11 @@ int QueryObject::ParseNode(const std::list::iterator &iter) LOGE("Only filter by keys in once!!"); return -E_INVALID_ARGS; } - - if (keys_.size() > DBConstant::MAX_BATCH_SIZE) { - return -E_MAX_LIMITS; - } hasInKeys_ = true; + int errCode = CheckInKeys(); + if (errCode != E_OK) { + return errCode; + } } return E_OK; } @@ -383,5 +385,24 @@ bool QueryObject::Empty() const { return queryObjNodes_.empty(); } + +int QueryObject::CheckInKeys() const +{ + if (!hasInKeys_) { + return E_OK; + } + if (keys_.empty()) { + return -E_INVALID_ARGS; + } + if (keys_.size() > DBConstant::MAX_BATCH_SIZE) { + return -E_MAX_LIMITS; + } + for (const auto &key : keys_) { + if (key.empty() || key.size() > DBConstant::MAX_KEY_SIZE) { + return -E_INVALID_ARGS; + } + } + return E_OK; +} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h index 6f732c209..812ef5c67 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h @@ -26,7 +26,8 @@ public: QueryObject(); explicit QueryObject(const Query &query); // for query sync - QueryObject(const std::list &queryObjNodes, const std::vector &prefixKey); + QueryObject(const std::list &queryObjNodes, const std::vector &prefixKey, + const std::set &keys); virtual ~QueryObject(); int Init(); SqliteQueryHelper GetQueryHelper(int &errCode); @@ -82,6 +83,7 @@ private: int CheckLinkerBefore(const std::list::iterator &iter) const; void ClearNodesFlag(); void GetAttrFromQueryObjNodes(); + int CheckInKeys() const; SchemaObject schema_; // used to check and parse schema filed int limit_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp index 31b551677..39b2152b4 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp @@ -84,8 +84,9 @@ int DeSerializeDataObjNode(Parcel &parcel, QueryObjNode &objNode) QuerySyncObject::QuerySyncObject() {} -QuerySyncObject::QuerySyncObject(const std::list &queryObjNodes, const std::vector &prefixKey) - : QueryObject(queryObjNodes, prefixKey) +QuerySyncObject::QuerySyncObject(const std::list &queryObjNodes, const std::vector &prefixKey, + const std::set &keys) + : QueryObject(queryObjNodes, prefixKey, keys) {} QuerySyncObject::QuerySyncObject(const Query &query) @@ -100,7 +101,7 @@ int QuerySyncObject::GetObjContext(ObjContext &objContext) const if (!isValid_) { return -E_INVALID_QUERY_FORMAT; } - objContext.version = isTableNameSpecified_ ? QUERY_SYNC_OBJECT_VERSION_1 : QUERY_SYNC_OBJECT_VERSION_0; + objContext.version = QUERY_SYNC_OBJECT_VERSION_CURRENT; objContext.prefixKey.assign(prefixKey_.begin(), prefixKey_.end()); objContext.suggestIndex = suggestIndex_; objContext.queryObjNodes = queryObjNodes_; @@ -114,9 +115,6 @@ std::string QuerySyncObject::GetIdentify() const } // suggestionIndex is local attribute, do not need to be propagated to remote uint64_t len = Parcel::GetVectorCharLen(prefixKey_); - if (isTableNameSpecified_) { - len += Parcel::GetStringLen(tableName_); - } for (const QueryObjNode &node : queryObjNodes_) { if (node.operFlag == QueryObjType::LIMIT || node.operFlag == QueryObjType::ORDERBY || node.operFlag == QueryObjType::SUGGEST_INDEX) { @@ -129,14 +127,22 @@ std::string QuerySyncObject::GetIdentify() const } } + // QUERY_SYNC_OBJECT_VERSION_1 added. + if (isTableNameSpecified_) { + len += Parcel::GetStringLen(tableName_); + } + if (!keys_.empty()) { + for (const auto &key : keys_) { + len += Parcel::GetVectorCharLen(key); + } + } + // QUERY_SYNC_OBJECT_VERSION_1 end. + std::vector buff(len, 0); // It will affect the hash result, the default value cannot be modified Parcel parcel(buff.data(), len); // The order needs to be consistent, otherwise it will affect the hash result (void)parcel.WriteVectorChar(prefixKey_); - if (isTableNameSpecified_) { - (void)parcel.WriteString(tableName_); - } for (const QueryObjNode &node : queryObjNodes_) { if (node.operFlag == QueryObjType::LIMIT || node.operFlag == QueryObjType::ORDERBY || node.operFlag == QueryObjType::SUGGEST_INDEX) { @@ -151,6 +157,17 @@ std::string QuerySyncObject::GetIdentify() const } } + // QUERY_SYNC_OBJECT_VERSION_1 added. + if (isTableNameSpecified_) { + (void)parcel.WriteString(tableName_); + } + if (!keys_.empty()) { + for (const auto &key : keys_) { + (void)parcel.WriteVectorChar(key); + } + } + // QUERY_SYNC_OBJECT_VERSION_1 end. + if (parcel.IsError()) { return std::string(); } @@ -184,9 +201,6 @@ int QuerySyncObject::SerializeData(Parcel &parcel, uint32_t softWareVersion) (void)parcel.WriteUInt32(context.version); (void)parcel.WriteVectorChar(context.prefixKey); (void)parcel.WriteString(context.suggestIndex); - if (isTableNameSpecified_) { - (void)parcel.WriteString(tableName_); - } (void)parcel.WriteUInt32(context.queryObjNodes.size()); parcel.EightByteAlign(); @@ -197,6 +211,18 @@ int QuerySyncObject::SerializeData(Parcel &parcel, uint32_t softWareVersion) return errCode; } } + + // QUERY_SYNC_OBJECT_VERSION_1 added. + (void)parcel.WriteUInt32(static_cast(isTableNameSpecified_)); + if (isTableNameSpecified_) { + (void)parcel.WriteString(tableName_); + } + (void)parcel.WriteUInt32(keys_.size()); + for (const auto &key : keys_) { + (void)parcel.WriteVectorChar(key); + } + // QUERY_SYNC_OBJECT_VERSION_1 end. + if (parcel.IsError()) { // parcel almost success return -E_INVALID_ARGS; } @@ -214,17 +240,13 @@ int QuerySyncObject::DeSerializeData(Parcel &parcel, QuerySyncObject &queryObj) ObjContext context; (void)parcel.ReadUInt32(context.version); - if (context.version > QUERY_SYNC_OBJECT_VERSION_1) { + if (context.version > QUERY_SYNC_OBJECT_VERSION_CURRENT) { LOGE("Parcel version and deserialize version not matched! ver=%u", context.version); return -E_VERSION_NOT_SUPPORT; } (void)parcel.ReadVectorChar(context.prefixKey); (void)parcel.ReadString(context.suggestIndex); - std::string tableName; - if (context.version == QUERY_SYNC_OBJECT_VERSION_1) { - parcel.ReadString(tableName); - } uint32_t nodesSize = 0; (void)parcel.ReadUInt32(nodesSize); @@ -241,11 +263,33 @@ int QuerySyncObject::DeSerializeData(Parcel &parcel, QuerySyncObject &queryObj) context.queryObjNodes.emplace_back(node); } + // QUERY_SYNC_OBJECT_VERSION_1 added. + std::string tableName; + std::set keys; + if (context.version >= QUERY_SYNC_OBJECT_VERSION_1) { + uint32_t isTblNameExist = 0; + (void)parcel.ReadUInt32(isTblNameExist); + if (isTblNameExist) { + (void)parcel.ReadString(tableName); + } + uint32_t keysSize = 0; + (void)parcel.ReadUInt32(keysSize); + if (keysSize > DBConstant::MAX_BATCH_SIZE) { + return -E_PARSE_FAIL; + } + for (uint32_t i = 0; i < keysSize; ++i) { + Key key; + (void)parcel.ReadVector(key); + keys.emplace(key); + } + } + // QUERY_SYNC_OBJECT_VERSION_1 end. + if (parcel.IsError()) { // almost success return -E_INVALID_ARGS; } - queryObj = QuerySyncObject(context.queryObjNodes, context.prefixKey); - if (context.version == QUERY_SYNC_OBJECT_VERSION_1) { + queryObj = QuerySyncObject(context.queryObjNodes, context.prefixKey, keys); + if (!tableName.empty()) { queryObj.SetTableName(tableName); } return E_OK; @@ -257,9 +301,6 @@ uint32_t QuerySyncObject::CalculateLen() const len += Parcel::GetUInt32Len(); // version len += Parcel::GetVectorCharLen(prefixKey_); len += Parcel::GetStringLen(suggestIndex_); - if (isTableNameSpecified_) { - len += Parcel::GetStringLen(tableName_); - } len += Parcel::GetUInt32Len(); // nodes size len = Parcel::GetEightByteAlign(len); for (const QueryObjNode &node : queryObjNodes_) { @@ -276,6 +317,18 @@ uint32_t QuerySyncObject::CalculateLen() const len += Parcel::GetInt64Len() + Parcel::GetStringLen(node.fieldValue[i].stringValue); } } + + // QUERY_SYNC_OBJECT_VERSION_1 added. + len += Parcel::GetUInt32Len(); // whether the table name exists. + if (isTableNameSpecified_) { + len += Parcel::GetStringLen(tableName_); + } + len += Parcel::GetUInt32Len(); // size of keys_ + for (const auto &key : keys_) { + len += Parcel::GetVectorCharLen(key); + } + // QUERY_SYNC_OBJECT_VERSION_1 end. + len = Parcel::GetEightByteAlign(len); if (len > INT32_MAX) { return 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h index 35d18750f..bb10d6ab9 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h @@ -24,6 +24,7 @@ namespace DistributedDB { const uint32_t QUERY_SYNC_OBJECT_VERSION_0 = 0; // for kvDB const uint32_t QUERY_SYNC_OBJECT_VERSION_1 = 1; // for relational DB, which need specify tableName +const uint32_t QUERY_SYNC_OBJECT_VERSION_CURRENT = QUERY_SYNC_OBJECT_VERSION_1; // always point the last. struct ObjContext { uint32_t version = QUERY_SYNC_OBJECT_VERSION_0; // serialized struct version @@ -36,7 +37,8 @@ struct ObjContext { class QuerySyncObject : public QueryObject { public: QuerySyncObject(); - QuerySyncObject(const std::list &queryObjNodes, const std::vector &prefixKey); + QuerySyncObject(const std::list &queryObjNodes, const std::vector &prefixKey, + const std::set &keys); explicit QuerySyncObject(const Query &query); ~QuerySyncObject() override; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp index 64e3dee73..9fc5eec1c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp @@ -278,7 +278,7 @@ int SqliteQueryHelper::GetQuerySql(std::string &sql, bool onlyRowid) const std::string &querySqlForUse = (onlyRowid ? PRE_QUERY_ROWID_SQL : PRE_QUERY_KV_SQL); sql = AssembleSqlForSuggestIndex(querySqlForUse, FILTER_NATIVE_DATA_SQL); sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); - sql = keys_.empty() ? sql : (sql + "AND " + MapKeysInToSql(keys_.size())); + sql = keys_.empty() ? sql : (sql + " AND " + MapKeysInToSql(keys_.size())); if (transformed_) { LOGD("This query object has been parsed."); sql += querySql_; @@ -302,7 +302,7 @@ int SqliteQueryHelper::GetSyncDataCheckSql(std::string &sql) } sql = PRE_QUERY_ITEM_SQL + tableName_ + " WHERE hash_key=? AND (flag&0x01=0) "; sql += hasPrefixKey_ ? " AND (key>=? AND key<=?) " : ""; - sql = keys_.empty() ? sql : (sql + "AND " + MapKeysInToSql(keys_.size())); + sql = keys_.empty() ? sql : (sql + " AND " + MapKeysInToSql(keys_.size())); if (!transformed_) { errCode = ToQuerySql(); if (errCode != E_OK) { @@ -357,7 +357,7 @@ int SqliteQueryHelper::GetCountQuerySql(std::string &sql) } sql = AssembleSqlForSuggestIndex(PRE_GET_COUNT_SQL, FILTER_NATIVE_DATA_SQL); sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); - sql = keys_.empty() ? sql : (sql + "AND " + MapKeysInToSql(keys_.size())); + sql = keys_.empty() ? sql : (sql + " AND " + MapKeysInToSql(keys_.size())); sql += countSql_; return E_OK; } @@ -383,6 +383,7 @@ int SqliteQueryHelper::GetQuerySqlStatement(sqlite3 *dbHandle, const std::string errCode = BindKeysToStmt(keys_, statement, index); if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); return errCode; } @@ -493,7 +494,7 @@ int SqliteQueryHelper::GetSyncDataQuerySql(std::string &sql, bool hasSubQuery) sql = AssembleSqlForSuggestIndex(PRE_QUERY_ITEM_SQL + tableName_ + " ", FILTER_REMOTE_QUERY); sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); - sql = keys_.empty() ? sql : (sql + "AND " + MapKeysInToSql(keys_.size())); + sql = keys_.empty() ? sql : (sql + " AND " + MapKeysInToSql(keys_.size())); sql = hasSubQuery ? sql : (sql + " AND (timestamp>=? AND timestamp &keys, sqlite3_stmt *&countStmt, int &index) const; + std::string MapKeysInSubCondition(const std::string &accessStr) const; // For InKeys. + SchemaObject schema_; std::list queryObjNodes_; std::vector prefixKey_; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp index aef048811..64ed867f0 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp @@ -1805,9 +1805,9 @@ HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeys001, TestSize.Level1) EXPECT_EQ(g_kvDelegateStatusForQuery, OK); Key key = {'1'}; - DBStatus status = g_kvDelegateCallbackForQuery->Put(key, VALUE_1); + DBStatus status = g_kvNbDelegatePtrForQuery->Put(key, VALUE_1); ASSERT_EQ(status, OK); - const int dataSize = 10; + const int dataSize = 10; // 10 data for test std::set keys; for (uint8_t i = 0; i < dataSize; i++) { key.push_back(i); @@ -1838,10 +1838,9 @@ HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeys001, TestSize.Level1) g_kvNbDelegatePtrForQuery->CloseResultSet(resultSet); /** - * @tc.steps: step3. Call GetEntries With Query, set all keys at Inkeys. + * @tc.steps: step3. Call GetEntries With Query, set one other key at Inkeys. * @tc.expected: step3. Returns KvStoreResultSet, the count is 0, */ - g_kvNbDelegatePtrForQuery->GetEntries() g_kvNbDelegatePtrForQuery->GetEntries(Query::Select().InKeys({KEY_7}), resultSet); ASSERT_NE(resultSet, nullptr); ASSERT_EQ(resultSet->GetCount(), 0); @@ -1851,12 +1850,12 @@ HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeys001, TestSize.Level1) } /** - * @tc.name: InKeysLimit001 - * @tc.desc: InKeys query limit verification - * @tc.type: FUNC - * @tc.require: AR000EPARK - * @tc.author: xushaohua - */ + * @tc.name: InKeysLimit001 + * @tc.desc: InKeys query limit verification + * @tc.type: FUNC + * @tc.require: AR000EPARK + * @tc.author: xushaohua + */ HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeysLimit001, TestSize.Level1) { /** @@ -1882,6 +1881,23 @@ HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeysLimit001, TestSize.Leve */ KvStoreResultSet *resultSet = nullptr; DBStatus status = g_kvNbDelegatePtrForQuery->GetEntries(Query::Select().InKeys(keys), resultSet); + EXPECT_EQ(status, OVER_MAX_LIMITS); + EXPECT_EQ(resultSet, nullptr); + + /** + * @tc.steps: step4. Call GetEntries With Query, set keys empty. + * @tc.expected: step4. Returns INVALID_ARGS, the resultSet is nullptr, + */ + status = g_kvNbDelegatePtrForQuery->GetEntries(Query::Select().InKeys({}), resultSet); + EXPECT_EQ(status, INVALID_ARGS); + EXPECT_EQ(resultSet, nullptr); + + /** + * @tc.steps: step4. Call GetEntries With Query, set a invalid key. + * @tc.expected: step4. Returns INVALID_ARGS, the resultSet is nullptr, + */ + status = g_kvNbDelegatePtrForQuery->GetEntries(Query::Select().InKeys({{}}), resultSet); + EXPECT_EQ(status, INVALID_ARGS); EXPECT_EQ(resultSet, nullptr); EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtrForQuery), OK); @@ -1889,20 +1905,20 @@ HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeysLimit001, TestSize.Leve } /** - * @tc.name: InKeysAndOthers001 - * @tc.desc: Combination of InKeys query and logical filtering - * @tc.type: FUNC - * @tc.require: AR000EPARK - * @tc.author: xushaohua - */ -HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeysAndOthers001, TestSize.Level1) + * @tc.name: InKeysAndOther001 + * @tc.desc: Combination of InKeys query and logical filtering + * @tc.type: FUNC + * @tc.require: AR000EPARK + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeysAndOther001, TestSize.Level1) { /** * @tc.steps: step1. Create a database And Preset Data */ KvStoreNbDelegate::Option option = {true, false, false}; option.schema = SCHEMA_DEFINE2; - g_mgr.GetKvStore("InKeysAndOthers001", option, g_kvNbDelegateCallbackForQuery); + g_mgr.GetKvStore("InKeysAndOther001", option, g_kvNbDelegateCallbackForQuery); ASSERT_NE(g_kvNbDelegatePtrForQuery, nullptr); EXPECT_EQ(g_kvDelegateStatusForQuery, OK); @@ -1924,26 +1940,26 @@ HWTEST_F(DistributedDBInterfacesDataOperationTest, InKeysAndOthers001, TestSize. * @tc.expected: step3. Returns OK */ query1 = Query::Select().InKeys(keys).EqualTo("$.field_name1", 1); - int errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); + errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); EXPECT_EQ(errCode, OK); /** - * @tc.steps: step4. Call GetEntries With Query, use EqualTo and InKeys and EqualTo, all valid + * @tc.steps: step4. Call GetEntries With Query, use EqualTo, InKeys and EqualTo, all valid * @tc.expected: step4. Returns OK */ query1 = Query::Select().EqualTo("$.field_name1", 1).InKeys(keys).And().EqualTo("$.field_name2", 2); - int errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); + errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); EXPECT_EQ(errCode, OK); /** - * @tc.steps: step4. Call GetEntries With Query, use EqualTo and InKeys and EqualTo, all valid - * @tc.expected: step4. Returns OK + * @tc.steps: step4. Call GetEntries With Query, use EqualTo, InKeys and EqualTo, has invalid + * @tc.expected: step4. Returns NOT_FOUND */ query1 = Query::Select().EqualTo("$.field_name1", 1).InKeys(keys).And().EqualTo("$.field_name1", 2); - int errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); + errCode = g_kvNbDelegatePtrForQuery->GetEntries(query1, entriesRes); EXPECT_EQ(errCode, NOT_FOUND); EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtrForQuery), OK); - EXPECT_EQ(g_mgr.DeleteKvStore("InKeysAndOthers001"), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("InKeysAndOther001"), OK); } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp index 5244086af..4ba3c999a 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp @@ -1127,7 +1127,7 @@ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest001, TestSize * @tc.steps:step1. Create a query object with table name is specified * @tc.expected: ok */ - Query query1 = Query::Select("Relatonal_table").EqualTo("field1", "abc"); + Query query1 = Query::Select("Relational_table").EqualTo("field1", "abc"); QuerySyncObject obj1(query1); /** @@ -1158,10 +1158,10 @@ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest001, TestSize */ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest002, TestSize.Level1) { - Query query1 = Query::Select("Relatonal_table1").EqualTo("field1", "abc"); + Query query1 = Query::Select("Relational_table1").EqualTo("field1", "abc"); QuerySyncObject obj1(query1); - Query query2 = Query::Select("Relatonal_table2").EqualTo("field1", "abc"); + Query query2 = Query::Select("Relational_table2").EqualTo("field1", "abc"); QuerySyncObject obj2(query2); /** @@ -1169,4 +1169,41 @@ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest002, TestSize * @tc.expected: identity should be different. */ EXPECT_NE(obj1.GetIdentify(), obj2.GetIdentify()); +} + +/** + * @tc.name: SerializeAndDeserializeForVer1 + * @tc.desc: Test querySyncObject serialization and deserialization. + * @tc.type: FUNC + * @tc.require: AR000GOHO7 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, SerializeAndDeserializeForVer1, TestSize.Level1) +{ + Query qeury1 = Query::Select("table1").EqualTo("field1", "abc").InKeys({KEY_1, KEY_2, KEY_3}); + QuerySyncObject obj1(qeury1); + + /** + * @tc.steps:step1. Serialize obj1. + * @tc.expected: Serialize successfully. + */ + auto len = obj1.CalculateParcelLen(SOFTWARE_VERSION_CURRENT); + std::vector buffer(len); + Parcel parcel1(buffer.data(), buffer.size()); + obj1.SerializeData(parcel1, SOFTWARE_VERSION_CURRENT); + ASSERT_EQ(parcel1.IsError(), false); + + /** + * @tc.steps:step2. Deserialize obj1. + * @tc.expected: Deserialize successfully. + */ + QuerySyncObject obj2; + Parcel parcel2(buffer.data(), buffer.size()); + ASSERT_EQ(parcel2.IsError(), false); + + /** + * @tc.steps:step3. check object identity + * @tc.expected: identity should be the same. + */ + EXPECT_NE(obj1.GetIdentify(), obj2.GetIdentify()); } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp index e9c5bc4af..9939f468d 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp @@ -463,6 +463,77 @@ HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, NormalSync004, TestSize.Level1) } } +/** + * @tc.name: NormalSync005 + * @tc.desc: Test normal push sync for inkeys query. + * @tc.type: FUNC + * @tc.require: AR000GOHO7 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, NormalSync005, TestSize.Level1) +{ + InitNormalDb(); + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step1. deviceA put K1-K5 + */ + ASSERT_EQ(g_kvDelegatePtr->PutBatch( + {{KEY_1, VALUE_1}, {KEY_2, VALUE_2}, {KEY_3, VALUE_3}, {KEY_4, VALUE_4}, {KEY_5, VALUE_5}}), OK); + + /** + * @tc.steps: step2. deviceA sync K2,K4 and wait + * @tc.expected: step2. sync should return OK. + */ + Query query = Query::Select().InKeys({KEY_2, KEY_4}); + std::map result; + ASSERT_EQ(g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result, query), OK); + + /** + * @tc.expected: step3. onComplete should be called. + */ + ASSERT_EQ(result.size(), devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_EQ(pair.second, OK); + } + + /** + * @tc.steps: step4. deviceB have K2K4 and have no K1K3K5. + * @tc.expected: step4. sync should return OK. + */ + VirtualDataItem item; + EXPECT_EQ(g_deviceB->GetData(KEY_2, item), E_OK); + EXPECT_EQ(item.value, VALUE_2); + EXPECT_EQ(g_deviceB->GetData(KEY_4, item), E_OK); + EXPECT_EQ(item.value, VALUE_4); + EXPECT_EQ(g_deviceB->GetData(KEY_1, item), -E_NOT_FOUND); + EXPECT_EQ(g_deviceB->GetData(KEY_3, item), -E_NOT_FOUND); + EXPECT_EQ(g_deviceB->GetData(KEY_5, item), -E_NOT_FOUND); + + /** + * @tc.steps: step5. deviceA sync with invalid inkeys query + * @tc.expected: step5. sync failed and the rc is right. + */ + query = Query::Select().InKeys({}); + result.clear(); + ASSERT_EQ(g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result, query), INVALID_ARGS); + + std::set keys; + for (uint8_t i = 0; i < DBConstant::MAX_BATCH_SIZE + 1; i++) { + Key key = { i }; + keys.emplace(key); + } + query = Query::Select().InKeys(keys); + result.clear(); + ASSERT_EQ(g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result, query), OVER_MAX_LIMITS); + + query = Query::Select().InKeys({{}}); + result.clear(); + ASSERT_EQ(g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result, query), INVALID_ARGS); +} + HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, QueryRequestPacketTest001, TestSize.Level1) { /** diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp index cd127f1eb..132d68b3a 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp @@ -79,6 +79,18 @@ namespace { "\"field_name8\":100," "\"field_name9\":100," "\"field_name10\":100}"; + + const std::string SCHEMA_VALUE2 = + "{\"field_name1\":false," + "\"field_name2\":true," + "\"field_name3\":100," + "\"field_name4\":200," + "\"field_name5\":3.14," + "\"field_name6\":\"3.1415\"," + "\"field_name7\":100," + "\"field_name8\":100," + "\"field_name9\":100," + "\"field_name10\":100}"; } class DistributedDBSingleVerP2PSubscribeSyncTest : public testing::Test { @@ -166,7 +178,6 @@ void InitSubSchemaDb() ASSERT_TRUE(g_schemaKvDelegatePtr != nullptr); } - void CheckUnFinishedMap(uint32_t sizeA, uint32_t sizeB, std::vector &deviceAQueies, std::vector &deviceBQueies, SubscribeManager &subManager) { @@ -725,4 +736,109 @@ HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeSync002, TestSize. LOGI("============step 4============"); Query query3 = Query::Select().EqualTo("$.field_name1", 1).OrderBy("$.field_name7"); EXPECT_TRUE(g_schemaKvDelegatePtr->SubscribeRemoteQuery(devices, nullptr, query3, true) == NOT_SUPPORT); +} + +/** + * @tc.name: subscribeSync003 + * @tc.desc: test subscribe sync with inkeys query + * @tc.type: FUNC + * @tc.require: AR000GOHO7 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeSync003, TestSize.Level1) +{ + /** + * @tc.steps: step1. InitSchemaDb + */ + LOGI("============step 1============"); + InitSubSchemaDb(); + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step2. deviceB subscribe inkeys(k2k4) query to deviceA + */ + LOGI("============step 2============"); + Query query = Query::Select().InKeys({KEY_2, KEY_4}); + g_deviceB->Subscribe(QuerySyncObject(query), true, 1); + + /** + * @tc.steps: step3. deviceA put k1-k5 and wait + */ + LOGI("============step 3============"); + EXPECT_EQ(OK, g_schemaKvDelegatePtr->PutBatch({ + {KEY_1, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, + {KEY_2, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, + {KEY_3, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, + {KEY_4, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, + {KEY_5, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, + })); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + + /** + * @tc.steps: step4. deviceB has k2k4, has no k1k3k5 + */ + LOGI("============step 4============"); + VirtualDataItem item; + EXPECT_EQ(g_deviceB->GetData(KEY_2, item), E_OK); + EXPECT_EQ(item.value, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())); + EXPECT_EQ(g_deviceB->GetData(KEY_4, item), E_OK); + EXPECT_EQ(item.value, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())); + EXPECT_EQ(g_deviceB->GetData(KEY_1, item), -E_NOT_FOUND); + EXPECT_EQ(g_deviceB->GetData(KEY_3, item), -E_NOT_FOUND); + EXPECT_EQ(g_deviceB->GetData(KEY_5, item), -E_NOT_FOUND); + + /** + * @tc.steps: step5. deviceB subscribe inkeys(k3k5) and equal to query to deviceA + */ + LOGI("============step 5============"); + query = Query::Select().InKeys({KEY_3, KEY_5}).EqualTo("$.field_name3", 100); // 100 for test. + g_deviceB->Subscribe(QuerySyncObject(query), true, 2); + + /** + * @tc.steps: step6. deviceA put k1v2,k3v2,k5v1 and wait + */ + LOGI("============step 6============"); + EXPECT_EQ(OK, g_schemaKvDelegatePtr->PutBatch({ + {KEY_1, Value(SCHEMA_VALUE2.begin(), SCHEMA_VALUE2.end())}, + {KEY_3, Value(SCHEMA_VALUE2.begin(), SCHEMA_VALUE2.end())}, + {KEY_5, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, + })); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + + /** + * @tc.steps: step7. deviceB has k3, has no k1k5 + */ + LOGI("============step 7============"); + EXPECT_EQ(g_deviceB->GetData(KEY_3, item), E_OK); + EXPECT_EQ(item.value, Value(SCHEMA_VALUE2.begin(), SCHEMA_VALUE2.end())); + EXPECT_EQ(g_deviceB->GetData(KEY_1, item), -E_NOT_FOUND); + EXPECT_EQ(g_deviceB->GetData(KEY_5, item), -E_NOT_FOUND); + + + /** + * @tc.steps: step8. deviceB subscribe inkeys(k1, key6) and prefix key "k" query to deviceA + */ + LOGI("============step 8============"); + Key key6 { 'k', '6' }; + query = Query::Select().InKeys({KEY_1, key6}).PrefixKey({ 'k' }); + g_deviceB->Subscribe(QuerySyncObject(query), true, 3); + + /** + * @tc.steps: step9. deviceA put k1,key6 and wait + */ + LOGI("============step 9============"); + EXPECT_EQ(OK, g_schemaKvDelegatePtr->PutBatch({ + {key6, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, + {KEY_1, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, + })); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + + /** + * @tc.steps: step10. deviceB has key6, has no k1 + */ + LOGI("============step 10============"); + EXPECT_EQ(g_deviceB->GetData(key6, item), E_OK); + EXPECT_EQ(item.value, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())); + EXPECT_EQ(g_deviceB->GetData(KEY_1, item), -E_NOT_FOUND); } \ No newline at end of file -- Gitee From 4bcf97a109210867e7a92ed8414db7ef281f71f2 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Tue, 18 Jan 2022 11:16:38 +0800 Subject: [PATCH 18/38] Fix issue Signed-off-by: lidwchn --- .../storage/src/sqlite/query_sync_object.cpp | 27 +++++++++++++------ .../storage/src/sqlite/query_sync_object.h | 5 ++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp index 39b2152b4..d8c33321b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp @@ -96,12 +96,21 @@ QuerySyncObject::QuerySyncObject(const Query &query) QuerySyncObject::~QuerySyncObject() {} +uint32_t QuerySyncObject::GetVersion() const +{ + uint32_t version = QUERY_SYNC_OBJECT_VERSION_0; + if (!tableName_.empty() || !keys_.empty()) { + version = QUERY_SYNC_OBJECT_VERSION_1; + } + return version; +} + int QuerySyncObject::GetObjContext(ObjContext &objContext) const { if (!isValid_) { return -E_INVALID_QUERY_FORMAT; } - objContext.version = QUERY_SYNC_OBJECT_VERSION_CURRENT; + objContext.version = GetVersion(); objContext.prefixKey.assign(prefixKey_.begin(), prefixKey_.end()); objContext.suggestIndex = suggestIndex_; objContext.queryObjNodes = queryObjNodes_; @@ -213,13 +222,15 @@ int QuerySyncObject::SerializeData(Parcel &parcel, uint32_t softWareVersion) } // QUERY_SYNC_OBJECT_VERSION_1 added. - (void)parcel.WriteUInt32(static_cast(isTableNameSpecified_)); - if (isTableNameSpecified_) { - (void)parcel.WriteString(tableName_); - } - (void)parcel.WriteUInt32(keys_.size()); - for (const auto &key : keys_) { - (void)parcel.WriteVectorChar(key); + if (context.version >= QUERY_SYNC_OBJECT_VERSION_1) { + (void)parcel.WriteUInt32(static_cast(isTableNameSpecified_)); + if (isTableNameSpecified_) { + (void)parcel.WriteString(tableName_); + } + (void)parcel.WriteUInt32(keys_.size()); + for (const auto &key : keys_) { + (void)parcel.WriteVectorChar(key); + } } // QUERY_SYNC_OBJECT_VERSION_1 end. diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h index bb10d6ab9..7d8ca7630 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h @@ -22,8 +22,8 @@ #include "parcel.h" namespace DistributedDB { -const uint32_t QUERY_SYNC_OBJECT_VERSION_0 = 0; // for kvDB -const uint32_t QUERY_SYNC_OBJECT_VERSION_1 = 1; // for relational DB, which need specify tableName +const uint32_t QUERY_SYNC_OBJECT_VERSION_0 = 0; +const uint32_t QUERY_SYNC_OBJECT_VERSION_1 = 1; // Add tableName_ and keys_. const uint32_t QUERY_SYNC_OBJECT_VERSION_CURRENT = QUERY_SYNC_OBJECT_VERSION_1; // always point the last. struct ObjContext { @@ -54,6 +54,7 @@ public: private: uint32_t CalculateLen() const; int GetObjContext(ObjContext &objContext) const; + uint32_t GetVersion() const; }; } #endif \ No newline at end of file -- Gitee From ac93a8563a4d4ac34ebdf2dd66857d138434141b Mon Sep 17 00:00:00 2001 From: lidwchn Date: Tue, 18 Jan 2022 11:37:47 +0800 Subject: [PATCH 19/38] Split overlong test case. Signed-off-by: lidwchn --- ...eddb_single_ver_p2p_subsribe_sync_test.cpp | 71 ++++++++++++++----- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp index 132d68b3a..0d7037fbc 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp @@ -773,7 +773,7 @@ HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeSync003, TestSize. {KEY_4, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, {KEY_5, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, })); - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); /** * @tc.steps: step4. deviceB has k2k4, has no k1k3k5 @@ -787,57 +787,94 @@ HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeSync003, TestSize. EXPECT_EQ(g_deviceB->GetData(KEY_1, item), -E_NOT_FOUND); EXPECT_EQ(g_deviceB->GetData(KEY_3, item), -E_NOT_FOUND); EXPECT_EQ(g_deviceB->GetData(KEY_5, item), -E_NOT_FOUND); +} +/** + * @tc.name: subscribeSync004 + * @tc.desc: test subscribe sync with inkeys query + * @tc.type: FUNC + * @tc.require: AR000GOHO7 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeSync004, TestSize.Level1) +{ /** - * @tc.steps: step5. deviceB subscribe inkeys(k3k5) and equal to query to deviceA + * @tc.steps: step1. InitSchemaDb */ - LOGI("============step 5============"); - query = Query::Select().InKeys({KEY_3, KEY_5}).EqualTo("$.field_name3", 100); // 100 for test. + LOGI("============step 1============"); + InitSubSchemaDb(); + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step2. deviceB subscribe inkeys(k3k5) and equal to query to deviceA + */ + LOGI("============step 2============"); + Query query = Query::Select().InKeys({KEY_3, KEY_5}).EqualTo("$.field_name3", 100); // 100 for test. g_deviceB->Subscribe(QuerySyncObject(query), true, 2); /** - * @tc.steps: step6. deviceA put k1v2,k3v2,k5v1 and wait + * @tc.steps: step3. deviceA put k1v2,k3v2,k5v1 and wait */ - LOGI("============step 6============"); + LOGI("============step 3============"); EXPECT_EQ(OK, g_schemaKvDelegatePtr->PutBatch({ {KEY_1, Value(SCHEMA_VALUE2.begin(), SCHEMA_VALUE2.end())}, {KEY_3, Value(SCHEMA_VALUE2.begin(), SCHEMA_VALUE2.end())}, {KEY_5, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, })); - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); /** - * @tc.steps: step7. deviceB has k3, has no k1k5 + * @tc.steps: step4. deviceB has k3, has no k1k5 */ - LOGI("============step 7============"); + LOGI("============step 4============"); + VirtualDataItem item; EXPECT_EQ(g_deviceB->GetData(KEY_3, item), E_OK); EXPECT_EQ(item.value, Value(SCHEMA_VALUE2.begin(), SCHEMA_VALUE2.end())); EXPECT_EQ(g_deviceB->GetData(KEY_1, item), -E_NOT_FOUND); EXPECT_EQ(g_deviceB->GetData(KEY_5, item), -E_NOT_FOUND); +} +/** + * @tc.name: subscribeSync005 + * @tc.desc: test subscribe sync with inkeys query + * @tc.type: FUNC + * @tc.require: AR000GOHO7 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeSync005, TestSize.Level1) +{ + /** + * @tc.steps: step1. InitSchemaDb + */ + LOGI("============step 1============"); + InitSubSchemaDb(); + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); /** - * @tc.steps: step8. deviceB subscribe inkeys(k1, key6) and prefix key "k" query to deviceA + * @tc.steps: step2. deviceB subscribe inkeys(k1, key6) and prefix key "k" query to deviceA */ - LOGI("============step 8============"); + LOGI("============step 2============"); Key key6 { 'k', '6' }; - query = Query::Select().InKeys({KEY_1, key6}).PrefixKey({ 'k' }); + Query query = Query::Select().InKeys({KEY_1, key6}).PrefixKey({ 'k' }); g_deviceB->Subscribe(QuerySyncObject(query), true, 3); /** - * @tc.steps: step9. deviceA put k1,key6 and wait + * @tc.steps: step3. deviceA put k1,key6 and wait */ - LOGI("============step 9============"); + LOGI("============step 3============"); EXPECT_EQ(OK, g_schemaKvDelegatePtr->PutBatch({ {key6, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, {KEY_1, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())}, })); - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); /** - * @tc.steps: step10. deviceB has key6, has no k1 + * @tc.steps: step4. deviceB has key6, has no k1 */ - LOGI("============step 10============"); + LOGI("============step 4============"); + VirtualDataItem item; EXPECT_EQ(g_deviceB->GetData(key6, item), E_OK); EXPECT_EQ(item.value, Value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end())); EXPECT_EQ(g_deviceB->GetData(KEY_1, item), -E_NOT_FOUND); -- Gitee From 2512b1257adc8d218e9589c8c55a097ba04bd890 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Tue, 18 Jan 2022 11:48:00 +0800 Subject: [PATCH 20/38] Fix issue. Signed-off-by: lidwchn --- .../libs/distributeddb/storage/src/sqlite/query_object.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp index ba23abef6..e54e88bc7 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp @@ -246,11 +246,11 @@ int QueryObject::ParseNode(const std::list::iterator &iter) LOGE("Only filter by keys in once!!"); return -E_INVALID_ARGS; } - hasInKeys_ = true; int errCode = CheckInKeys(); if (errCode != E_OK) { return errCode; } + hasInKeys_ = true; } return E_OK; } @@ -388,9 +388,6 @@ bool QueryObject::Empty() const int QueryObject::CheckInKeys() const { - if (!hasInKeys_) { - return E_OK; - } if (keys_.empty()) { return -E_INVALID_ARGS; } -- Gitee From c6d17021f6456326ed952221cf78c8cf20e6f1ed Mon Sep 17 00:00:00 2001 From: zqq Date: Mon, 17 Jan 2022 21:23:50 +0800 Subject: [PATCH 21/38] add rdb ability sync support Signed-off-by: zqq --- .../distributeddb/common/include/db_errno.h | 1 + .../relational/relational_schema_object.h | 14 +- .../relational/relational_schema_object.cpp | 35 +- .../interfaces/src/relational/data_value.cpp | 11 +- .../relational/relational_sync_able_storage.h | 4 +- .../storage/src/data_transformer.cpp | 2 - .../relational/sqlite_relational_store.cpp | 15 +- .../distributeddb/syncer/src/ability_sync.cpp | 347 ++++++---- .../distributeddb/syncer/src/ability_sync.h | 53 +- .../distributeddb/syncer/src/isync_engine.h | 2 + .../syncer/src/single_ver_data_sync.cpp | 7 +- .../src/single_ver_kv_sync_task_context.cpp | 14 + .../src/single_ver_kv_sync_task_context.h | 4 + ...ingle_ver_relational_sync_task_context.cpp | 10 + .../single_ver_relational_sync_task_context.h | 5 + .../src/single_ver_relational_syncer.cpp | 20 +- .../syncer/src/single_ver_relational_syncer.h | 4 + .../src/single_ver_sync_state_machine.cpp | 1 + .../src/single_ver_sync_task_context.cpp | 14 - .../syncer/src/single_ver_sync_task_context.h | 5 +- .../distributeddb/syncer/src/sync_engine.cpp | 15 + .../distributeddb/syncer/src/sync_engine.h | 2 + .../syncer/src/sync_operation.cpp | 1 + .../distributeddb/syncer/src/sync_operation.h | 1 + ...stributeddb_interfaces_data_value_test.cpp | 201 ++++++ .../distributeddb_data_transformer_test.cpp | 287 +++++++++ ...ributeddb_relational_ver_p2p_sync_test.cpp | 595 ++++++++++++++---- .../syncer/relational_virtual_device.cpp | 10 + .../common/syncer/relational_virtual_device.h | 2 + .../common/syncer/virtual_communicator.cpp | 2 +- ...rtual_relational_ver_sync_db_interface.cpp | 19 +- ...virtual_relational_ver_sync_db_interface.h | 4 + 32 files changed, 1365 insertions(+), 342 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_value_test.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_data_transformer_test.cpp diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h index d76ab568f..b3c6acc35 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h @@ -142,6 +142,7 @@ constexpr int E_VALUE_MISMATCH_OTHER_REASON = (E_BASE + 185); // Value mismatch // Message with errorNo of Feedback-type is generated by CommunicatorAggregator without data part(No deserial if exist) constexpr int E_FEEDBACK_UNKNOWN_MESSAGE = (E_BASE + 200); // Unknown message feedback from remote device constexpr int E_FEEDBACK_COMMUNICATOR_NOT_FOUND = (E_BASE + 201); // Communicator not found feedback from remote device +constexpr int E_SCHEMA_CHANGE = (E_BASE + 202); // Schema has change when do sync } // namespace DistributedDB #endif // DISTRIBUTEDDB_ERRNO_H diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index 1a1f6cf92..6ccde2d3c 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -96,10 +96,11 @@ private: class RelationalSyncOpinion { public: - int CalculateParcelLen(uint32_t softWareVersion) const; - int Serialize(Parcel &parcel, uint32_t softWareVersion) const; - int Deserialization(const Parcel &parcel); - const SyncOpinion &GetTableOpinion(const std::string& tableName) const; + uint32_t CalculateParcelLen(uint32_t softWareVersion) const; + int SerializeData(Parcel &parcel, uint32_t softWareVersion) const; + static int DeserializeData(Parcel &parcel, RelationalSyncOpinion &opinion); + SyncOpinion GetTableOpinion(const std::string& tableName) const; + const std::map &GetOpinions() const; void AddSyncOpinion(const std::string &tableName, const SyncOpinion &opinion); private: std::map opinions_; @@ -107,7 +108,7 @@ private: class RelationalSyncStrategy { public: - const SyncStrategy &GetTableStrategy(const std::string &tableName) const; + SyncStrategy GetTableStrategy(const std::string &tableName) const; void AddSyncStrategy(const std::string &tableName, const SyncStrategy &strategy); private: std::map strategies_; @@ -119,8 +120,7 @@ public: ~RelationalSchemaObject() override = default; static RelationalSyncOpinion MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, - const std::string &remoteSchema, uint8_t remoteSchemaType); - + const std::string &remoteSchemaStr, uint8_t remoteSchemaType); // The remoteOpinion.checkOnReceive is ignored static RelationalSyncStrategy ConcludeSyncStrategy(const RelationalSyncOpinion &localOpinion, const RelationalSyncOpinion &remoteOpinion); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index b500194ad..478a3e6af 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -321,24 +321,33 @@ std::string TableInfo::ToTableInfoString() const return attrStr; } -int RelationalSyncOpinion::CalculateParcelLen(uint32_t softWareVersion) const +uint32_t RelationalSyncOpinion::CalculateParcelLen(uint32_t softWareVersion) const { return E_OK; } -int RelationalSyncOpinion::Serialize(Parcel &parcel, uint32_t softWareVersion) const +int RelationalSyncOpinion::SerializeData(Parcel &parcel, uint32_t softWareVersion) const { return E_OK; } -int RelationalSyncOpinion::Deserialization(const Parcel &parcel) +int RelationalSyncOpinion::DeserializeData(Parcel &parcel, RelationalSyncOpinion &opinion) { return E_OK; } -const SyncOpinion &RelationalSyncOpinion::GetTableOpinion(const std::string &tableName) const +SyncOpinion RelationalSyncOpinion::GetTableOpinion(const std::string &tableName) const { - return opinions_.at(tableName); + auto it = opinions_.find(tableName); + if (it == opinions_.end()) { + return {}; + } + return it->second; +} + +const std::map &RelationalSyncOpinion::GetOpinions() const +{ + return opinions_; } void RelationalSyncOpinion::AddSyncOpinion(const std::string &tableName, const SyncOpinion &opinion) @@ -346,9 +355,13 @@ void RelationalSyncOpinion::AddSyncOpinion(const std::string &tableName, const S opinions_[tableName] = opinion; } -const SyncStrategy &RelationalSyncStrategy::GetTableStrategy(const std::string &tableName) const +SyncStrategy RelationalSyncStrategy::GetTableStrategy(const std::string &tableName) const { - return strategies_.at(tableName); + auto it = strategies_.find(tableName); + if (it == strategies_.end()) { + return {}; + } + return it->second; } void RelationalSyncStrategy::AddSyncStrategy(const std::string &tableName, const SyncStrategy &strategy) @@ -356,14 +369,14 @@ void RelationalSyncStrategy::AddSyncStrategy(const std::string &tableName, const strategies_[tableName] = strategy; } -RelationalSyncOpinion RelationalSchemaObject::MakeLocalSyncOpinion( - const RelationalSchemaObject &localSchema, const std::string &remoteSchema, uint8_t remoteSchemaType) +RelationalSyncOpinion RelationalSchemaObject::MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, + const std::string &remoteSchemaStr, uint8_t remoteSchemaType) { return RelationalSyncOpinion(); } -RelationalSyncStrategy RelationalSchemaObject::ConcludeSyncStrategy( - const RelationalSyncOpinion &localOpinion, const RelationalSyncOpinion &remoteOpinion) +RelationalSyncStrategy RelationalSchemaObject::ConcludeSyncStrategy(const RelationalSyncOpinion &localOpinion, + const RelationalSyncOpinion &remoteOpinion) { return RelationalSyncStrategy(); } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index 3a9051c6e..00cd229dc 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -98,17 +98,8 @@ DataValue::DataValue(const DataValue &dataValue) } DataValue::DataValue(DataValue &&dataValue) noexcept - :type_(dataValue.type_), value_(dataValue.value_) { - switch (type_) { - case StorageType::STORAGE_TYPE_BLOB: - case StorageType::STORAGE_TYPE_TEXT: - dataValue.value_.blobPtr = nullptr; - break; - default: - break; - } - dataValue.ResetValue(); + *this = std::move(dataValue); } DataValue &DataValue::operator=(const DataValue &dataValue) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index ecf584fd9..721eb8961 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -17,6 +17,8 @@ #ifdef RELATIONAL_STORE #include "relational_db_sync_interface.h" +#include "relationaldb_properties.h" +#include "runtime_context.h" #include "sqlite_single_relational_storage_engine.h" #include "sqlite_single_ver_relational_continue_token.h" @@ -108,7 +110,7 @@ public: int RegisterSchemaChangedCallback(const std::function &callback) override; void NotifySchemaChanged(); - + void RegisterHeartBeatListener(const std::function &listener); int GetCompressionAlgo(std::set &algorithmSet) const override; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp index eb30a61f1..6705ecb4f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -338,8 +338,6 @@ int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalDa std::vector valueList; for (const auto &fieldInfo : remoteFieldInfo) { DataValue dataValue; - LOGD("[DataTransformer][DeSerializeValue] start deSerialize %s type %d", - fieldInfo.GetFieldName().c_str(), fieldInfo.GetStorageType()); uint32_t type = 0; parcel.ReadUInt32(type); auto iter = typeFuncMap.find(static_cast(type)); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index b516ab606..f012cca09 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -482,17 +482,6 @@ int SQLiteRelationalStore::RegisterLifeCycleCallback(const DatabaseLifeCycleNoti int errCode; { std::lock_guard lock(lifeCycleMutex_); - if (!notifier) { - if (lifeTimerId_ == 0) { - return E_OK; - } - errCode = StopLifeCycleTimer(); - if (errCode != E_OK) { - LOGE("Stop the life cycle timer failed:%d", errCode); - } - return E_OK; - } - if (lifeTimerId_ != 0) { errCode = StopLifeCycleTimer(); if (errCode != E_OK) { @@ -500,6 +489,10 @@ int SQLiteRelationalStore::RegisterLifeCycleCallback(const DatabaseLifeCycleNoti return errCode; } } + + if (!notifier) { + return E_OK; + } errCode = StartLifeCycleTimer(notifier); if (errCode != E_OK) { LOGE("Register life cycle timer failed:%d", errCode); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp index 5efe7b6d7..ba65b65ed 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp @@ -23,8 +23,10 @@ #include "db_common.h" #include "single_ver_kvdb_sync_interface.h" #include "single_ver_sync_task_context.h" +#include "single_ver_kv_sync_task_context.h" #ifdef RELATIONAL_STORE #include "relational_db_sync_interface.h" +#include "single_ver_relational_sync_task_context.h" #endif namespace DistributedDB { @@ -295,6 +297,7 @@ uint32_t AbilitySyncAckPacket::CalculateLen() const len += Parcel::GetUInt32Len(); // requirePeerConvert_ len += Parcel::GetUInt64Len(); // dbCreateTime_ len += DbAbility::CalculateLen(dbAbility_); // dbAbility_ + len += relationalSyncOpinion_.CalculateParcelLen(softwareVersion_); if (len > INT32_MAX) { LOGE("[AbilitySyncAckPacket][CalculateLen] err len:%llu", len); return 0; @@ -312,6 +315,16 @@ void AbilitySyncAckPacket::SetDbAbility(const DbAbility &dbAbility) dbAbility_ = dbAbility; } +void AbilitySyncAckPacket::SetRelationalSyncOpinion(const RelationalSyncOpinion &relationalSyncOpinion) +{ + relationalSyncOpinion_ = relationalSyncOpinion; +} + +RelationalSyncOpinion AbilitySyncAckPacket::GetRelationalSyncOpinion() const +{ + return relationalSyncOpinion_; +} + AbilitySync::AbilitySync() : communicator_(nullptr), storageInterface_(nullptr), @@ -387,8 +400,14 @@ int AbilitySync::AckRecv(const Message *message, ISyncTaskContext *context) std::string schema = packet->GetSchema(); if (remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_2_0) { HandleVersionV3AckSecOptionParam(packet, context); - SyncOpinion localSyncOpinion = HandleVersionV3AckSchemaParam(packet, schema, context); - bool permitSync = ((static_cast(context))->GetSyncStrategy()).permitSync; + AbilitySyncAckPacket ackPacket; + int errCode = HandleVersionV3AckSchemaParam(packet, ackPacket, context, true); + if (errCode != E_OK) { + return errCode; + } + auto singleVerContext = static_cast(context); + auto query = singleVerContext->GetQuery(); + bool permitSync = (singleVerContext->GetSyncStrategy(query)).permitSync; if (!permitSync) { (static_cast(context))->SetTaskErrCode(-E_SCHEMA_MISMATCH); LOGE("[AbilitySync][AckRecv] scheme check failed"); @@ -403,13 +422,10 @@ int AbilitySync::AckRecv(const Message *message, ISyncTaskContext *context) } DbAbility remoteDbAbility = packet->GetDbAbility(); (static_cast(context))->SetDbAbility(remoteDbAbility); - SendAck(message, localSyncOpinion, AbilitySync::CHECK_SUCCESS, true); + (void)SendAck(message, AbilitySync::CHECK_SUCCESS, true, ackPacket); (static_cast(context))->SetIsSchemaSync(true); } else { - bool isCompatible = true; - if (IsSingleKvVer()) { - isCompatible = static_cast(storageInterface_)->CheckCompatible(schema); - } + bool isCompatible = static_cast(storageInterface_)->CheckCompatible(schema); if (!isCompatible) { (static_cast(context))->SetTaskErrCode(-E_SCHEMA_MISMATCH); LOGE("[AbilitySync][AckRecv] scheme check failed"); @@ -430,54 +446,21 @@ int AbilitySync::RequestRecv(const Message *message, ISyncTaskContext *context) if (packet == nullptr) { return -E_INVALID_ARGS; } - SyncOpinion localSyncOpinion; if (packet->GetSendCode() == -E_VERSION_NOT_SUPPORT) { - SendAck(message, localSyncOpinion, -E_VERSION_NOT_SUPPORT, false); + AbilitySyncAckPacket ackPacket; + (void)SendAck(message, -E_VERSION_NOT_SUPPORT, false, ackPacket); LOGI("[AbilitySync][RequestRecv] version can not support, remote version is %u", packet->GetProtocolVersion()); return -E_VERSION_NOT_SUPPORT; } std::string schema = packet->GetSchema(); - bool isCompatible = true; - if (IsSingleKvVer()) { - isCompatible = static_cast(storageInterface_)->CheckCompatible(schema); - } + bool isCompatible = static_cast(storageInterface_)->CheckCompatible(schema); if (!isCompatible) { (static_cast(context))->SetTaskErrCode(-E_SCHEMA_MISMATCH); } uint32_t remoteSoftwareVersion = packet->GetSoftwareVersion(); context->SetRemoteSoftwareVersion(remoteSoftwareVersion); - int ackCode; - if (remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_2_0) { - localSyncOpinion = HandleVersionV3RequestParam(packet, context, schema); - if (SecLabelCheck(packet)) { - ackCode = E_OK; - } else { - ackCode = -E_SECURITY_OPTION_CHECK_ERROR; - } - } else { - LOGI("[AbilitySync][RequestRecv] remote version = %u, CheckSchemaCompatible = %d", - remoteSoftwareVersion, isCompatible); - return SendAck(message, SchemaObject(), E_OK, localSyncOpinion, false); - } - if (ackCode == E_OK && remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_3_0) { - ackCode = metadata_->SetDbCreateTime(deviceId_, packet->GetDbCreateTime(), true); - } - LOGI("[AbilitySync][RequestRecv] remote dev=%s,ver=%u,schemaCompatible=%d", STR_MASK(deviceId_), - remoteSoftwareVersion, isCompatible); - return SendAck(message, localSyncOpinion, ackCode, false); -} - -int AbilitySync::SendAck(const Message *message, SyncOpinion &localSyncOpinion, int ackCode, bool isAckNotify) -{ -#ifdef RELATIONAL_STORE - if (IsSingleRelationalVer()) { - SchemaObject schemaObject; - return SendAck(message, schemaObject, ackCode, localSyncOpinion, isAckNotify); - } -#endif - SchemaObject schemaObject = static_cast(storageInterface_)->GetSchemaInfo(); - return SendAck(message, schemaObject, ackCode, localSyncOpinion, isAckNotify); + return HandleRequestRecv(message, context, isCompatible); } int AbilitySync::AckNotifyRecv(const Message *message, ISyncTaskContext *context) @@ -502,12 +485,17 @@ int AbilitySync::AckNotifyRecv(const Message *message, ISyncTaskContext *context std::string schema = packet->GetSchema(); uint32_t remoteSoftwareVersion = packet->GetSoftwareVersion(); context->SetRemoteSoftwareVersion(remoteSoftwareVersion); - SyncOpinion localSyncOpinion = HandleVersionV3AckSchemaParam(packet, schema, context); + AbilitySyncAckPacket sendPacket; + errCode = HandleVersionV3AckSchemaParam(packet, sendPacket, context, false); + int ackCode = errCode; LOGI("[AckNotifyRecv] receive dev = %s ack notify, remoteSoftwareVersion = %u, ackCode = %d", STR_MASK(deviceId_), remoteSoftwareVersion, errCode); - (static_cast(context))->SetIsSchemaSync(true); - (void)SendAck(message, SchemaObject(), AbilitySync::LAST_NOTIFY, localSyncOpinion, true); - return E_OK; + if (errCode == E_OK) { + (static_cast(context))->SetIsSchemaSync(true); + ackCode = AbilitySync::LAST_NOTIFY; + } + (void)SendAckWithEmptySchema(message, ackCode, true); + return errCode; } bool AbilitySync::GetAbilitySyncFinishedStatus() const @@ -546,27 +534,17 @@ bool AbilitySync::SecLabelCheck(const AbilitySyncRequestPacket *packet) const } } -SyncOpinion AbilitySync::HandleVersionV3RequestParam(const AbilitySyncRequestPacket *packet, ISyncTaskContext *context, +void AbilitySync::HandleVersionV3RequestParam(const AbilitySyncRequestPacket *packet, ISyncTaskContext *context, const std::string &remoteSchema) const { int32_t remoteSecLabel = packet->GetSecLabel(); int32_t remoteSecFlag = packet->GetSecFlag(); - SecurityOption secOption = {remoteSecLabel, remoteSecFlag}; DbAbility remoteDbAbility = packet->GetDbAbility(); (static_cast(context))->SetDbAbility(remoteDbAbility); - (static_cast(context))->SetRemoteSeccurityOption(secOption); + (static_cast(context))->SetRemoteSeccurityOption({remoteSecLabel, remoteSecFlag}); (static_cast(context))->SetReceivcPermitCheck(false); -#ifdef RELATIONAL_STORE - if (IsSingleRelationalVer()) { - return SyncOpinion{true, false, false}; - } -#endif - uint8_t remoteSchemaType = packet->GetSchemaType(); - SchemaObject localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); - SyncOpinion localSyncOpinion = SchemaObject::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); LOGI("[AbilitySync][HandleVersionV3RequestParam] remoteSecLabel = %d, remoteSecFlag = %d, remoteSchemaType = %u", - remoteSecLabel, remoteSecFlag, remoteSchemaType); - return localSyncOpinion; + remoteSecLabel, remoteSecFlag, packet->GetSchemaType()); } void AbilitySync::HandleVersionV3AckSecOptionParam(const AbilitySyncAckPacket *packet, @@ -580,26 +558,14 @@ void AbilitySync::HandleVersionV3AckSecOptionParam(const AbilitySyncAckPacket *p LOGI("[AbilitySync][AckRecv] remoteSecLabel = %d, remoteSecFlag = %d", remoteSecLabel, remoteSecFlag); } -SyncOpinion AbilitySync::HandleVersionV3AckSchemaParam(const AbilitySyncAckPacket *packet, - const std::string &remoteSchema, ISyncTaskContext *context) const +int AbilitySync::HandleVersionV3AckSchemaParam(const AbilitySyncAckPacket *recvPacket, + AbilitySyncAckPacket &sendPacket, ISyncTaskContext *context, bool sendOpinion) const { - bool permitSync = static_cast(packet->GetPermitSync()); - bool requirePeerConvert = static_cast(packet->GetRequirePeerConvert()); - SyncOpinion remoteOpinion = {permitSync, requirePeerConvert, true}; - uint8_t remoteSchemaType = packet->GetSchemaType(); -#ifdef RELATIONAL_STORE if (IsSingleRelationalVer()) { - auto localOpinion = SyncOpinion{true, false, false}; - SyncStrategy localStrategy = SchemaObject::ConcludeSyncStrategy(localOpinion, remoteOpinion); - (static_cast(context))->SetSyncStrategy(localStrategy); - return localOpinion; + return HandleRelationAckSchemaParam(recvPacket, sendPacket, context, sendOpinion); } -#endif - SchemaObject localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); - SyncOpinion localOpinion = SchemaObject::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); - SyncStrategy localStrategy = SchemaObject::ConcludeSyncStrategy(localOpinion, remoteOpinion); - (static_cast(context))->SetSyncStrategy(localStrategy); - return localOpinion; + HandleKvAckSchemaParam(recvPacket, context, sendPacket); + return E_OK; } void AbilitySync::GetPacketSecOption(SecurityOption &option) const @@ -698,44 +664,6 @@ int AbilitySync::DeSerialization(const uint8_t *buffer, uint32_t length, Message } } -#ifdef RELATIONAL_STORE -int AbilitySync::SendAck(const Message *inMsg, const ISchema &schemaObj, int ackCode, SyncOpinion localOpinion, - bool isAckNotify) -#else -int AbilitySync::SendAck(const Message *inMsg, const SchemaObject &schemaObj, int ackCode, SyncOpinion localOpinion, - bool isAckNotify) -#endif -{ - AbilitySyncAckPacket ackPacket; - int errCode = SetAbilityAckBodyInfo(ackPacket, schemaObj, ackCode, localOpinion, isAckNotify); - if (errCode != E_OK) { - return errCode; - } - Message *ackMessage = new (std::nothrow) Message(ABILITY_SYNC_MESSAGE); - if (ackMessage == nullptr) { - LOGE("[AbilitySync][SendAck] message create failed, may be memleak!"); - return -E_OUT_OF_MEMORY; - } - errCode = ackMessage->SetCopiedObject<>(ackPacket); - if (errCode != E_OK) { - LOGE("[AbilitySync][SendAck] SetCopiedObject failed, err %d", errCode); - delete ackMessage; - ackMessage = nullptr; - return errCode; - } - (!isAckNotify) ? ackMessage->SetMessageType(TYPE_RESPONSE) : ackMessage->SetMessageType(TYPE_NOTIFY); - ackMessage->SetTarget(deviceId_); - ackMessage->SetSessionId(inMsg->GetSessionId()); - ackMessage->SetSequenceId(inMsg->GetSequenceId()); - errCode = communicator_->SendMessage(deviceId_, ackMessage, false, SEND_TIME_OUT); - if (errCode != E_OK) { - LOGE("[AbilitySync][SendAck] SendPacket failed, err %d", errCode); - delete ackMessage; - ackMessage = nullptr; - } - return errCode; -} - int AbilitySync::RequestPacketCalculateLen(const Message *inMsg, uint32_t &len) { const AbilitySyncRequestPacket *packet = inMsg->GetObject(); @@ -803,6 +731,10 @@ int AbilitySync::AckPacketSerialization(uint8_t *buffer, uint32_t length, const if (parcel.IsError() || errCode != E_OK) { return -E_PARSE_FAIL; } + errCode = packet->GetRelationalSyncOpinion().SerializeData(parcel, SOFTWARE_VERSION_CURRENT); + if (parcel.IsError() || errCode != E_OK) { + return -E_PARSE_FAIL; + } return E_OK; } @@ -915,6 +847,13 @@ int AbilitySync::AckPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncAck return errCode; } packet->SetDbAbility(remoteDbAbility); + RelationalSyncOpinion relationalSyncOpinion; + errCode = RelationalSyncOpinion::DeserializeData(parcel, relationalSyncOpinion); + if (errCode != E_OK) { + LOGE("[AbilitySync] ack packet DeSerializ RelationalSyncOpinion failed."); + return errCode; + } + packet->SetRelationalSyncOpinion(relationalSyncOpinion); return E_OK; } @@ -986,14 +925,11 @@ int AbilitySync::SetAbilityRequestBodyInfo(AbilitySyncRequestPacket &packet, uin SchemaObject schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); schemaStr = schemaObj.ToSchemaString(); schemaType = static_cast(schemaObj.GetSchemaType()); - } -#ifdef RELATIONAL_STORE - if (IsSingleRelationalVer()) { + } else if (IsSingleRelationalVer()) { auto schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); schemaStr = schemaObj.ToSchemaString(); - schemaType = 0; + schemaType = static_cast(schemaObj.GetSchemaType()); } -#endif DbAbility dbAbility; errCode = GetDbAbilityInfo(dbAbility); if (errCode != E_OK) { @@ -1020,19 +956,11 @@ int AbilitySync::SetAbilityRequestBodyInfo(AbilitySyncRequestPacket &packet, uin return E_OK; } -#ifdef RELATIONAL_STORE -int AbilitySync::SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const ISchema &schemaObj, int ackCode, - SyncOpinion localOpinion, bool isAckNotify) -#else -int AbilitySync::SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const SchemaObject &schemaObj, int ackCode, - SyncOpinion localOpinion, bool isAckNotify) -#endif +int AbilitySync::SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, int ackCode, bool isAckNotify) const { int errCode = E_OK; ackPacket.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); ackPacket.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); - ackPacket.SetSchema(schemaObj.ToSchemaString()); - ackPacket.SetSchemaType(static_cast(schemaObj.GetSchemaType())); if (!isAckNotify) { SecurityOption option; GetPacketSecOption(option); @@ -1055,9 +983,19 @@ int AbilitySync::SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const Sc ackPacket.SetDbAbility(dbAbility); } ackPacket.SetAckCode(ackCode); + return E_OK; +} + +void AbilitySync::SetAbilityAckSchemaInfo(AbilitySyncAckPacket &ackPacket, const ISchema &schemaObj) const +{ + ackPacket.SetSchema(schemaObj.ToSchemaString()); + ackPacket.SetSchemaType(static_cast(schemaObj.GetSchemaType())); +} + +void AbilitySync::SetAbilityAckSyncOpinionInfo(AbilitySyncAckPacket &ackPacket, SyncOpinion localOpinion) const +{ ackPacket.SetPermitSync(localOpinion.permitSync); ackPacket.SetRequirePeerConvert(localOpinion.requirePeerConvert); - return E_OK; } int AbilitySync::GetDbAbilityInfo(DbAbility &dbAbility) const @@ -1107,10 +1045,153 @@ bool AbilitySync::IsSingleKvVer() const { return storageInterface_->GetInterfaceType() == ISyncInterface::SYNC_SVD; } -#ifdef RELATIONAL_STORE bool AbilitySync::IsSingleRelationalVer() const { +#ifdef RELATIONAL_STORE return storageInterface_->GetInterfaceType() == ISyncInterface::SYNC_RELATION; -} +#elif + return false; #endif +} + +int AbilitySync::HandleRequestRecv(const Message *message, ISyncTaskContext *context, bool isCompatible) +{ + const AbilitySyncRequestPacket *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + uint32_t remoteSoftwareVersion = packet->GetSoftwareVersion(); + int ackCode; + std::string schema = packet->GetSchema(); + if (remoteSoftwareVersion <= SOFTWARE_VERSION_RELEASE_2_0) { + LOGI("[AbilitySync][RequestRecv] remote version = %u, CheckSchemaCompatible = %d", + remoteSoftwareVersion, isCompatible); + return SendAckWithEmptySchema(message, E_OK, false); + } + HandleVersionV3RequestParam(packet, context, schema); + if (SecLabelCheck(packet)) { + ackCode = E_OK; + } else { + ackCode = -E_SECURITY_OPTION_CHECK_ERROR; + } + if (ackCode == E_OK && remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_3_0) { + ackCode = metadata_->SetDbCreateTime(deviceId_, packet->GetDbCreateTime(), true); + } + AbilitySyncAckPacket ackPacket; + if (IsSingleRelationalVer()) { + ackPacket.SetRelationalSyncOpinion(MakeRelationSyncOpnion(packet, schema)); + } else { + SetAbilityAckSyncOpinionInfo(ackPacket, MakeKvSyncOpnion(packet, schema)); + } + LOGI("[AbilitySync][RequestRecv] remote dev=%s,ver=%u,schemaCompatible=%d", STR_MASK(deviceId_), + remoteSoftwareVersion, isCompatible); + return SendAck(message, ackCode, false, ackPacket); +} + +int AbilitySync::SendAck(const Message *message, int ackCode, bool isAckNotify, AbilitySyncAckPacket &ackPacket) +{ + int errCode = SetAbilityAckBodyInfo(ackPacket, ackCode, isAckNotify); + if (errCode != E_OK) { + return errCode; + } + if (IsSingleRelationalVer()) { + auto schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); + SetAbilityAckSchemaInfo(ackPacket, schemaObj); + } else if(IsSingleKvVer()) { + SchemaObject schemaObject = static_cast(storageInterface_)->GetSchemaInfo(); + SetAbilityAckSchemaInfo(ackPacket, schemaObject); + } + return SendAck(message, ackPacket, isAckNotify); +} + +int AbilitySync::SendAckWithEmptySchema(const Message *message, int ackCode, + bool isAckNotify) +{ + AbilitySyncAckPacket ackPacket; + int errCode = SetAbilityAckBodyInfo(ackPacket, ackCode, isAckNotify); + if (errCode != E_OK) { + return errCode; + } + SetAbilityAckSchemaInfo(ackPacket, SchemaObject()); + return SendAck(message, ackPacket, isAckNotify); +} + +int AbilitySync::SendAck(const Message *inMsg, const AbilitySyncAckPacket &ackPacket, bool isAckNotify) +{ + Message *ackMessage = new (std::nothrow) Message(ABILITY_SYNC_MESSAGE); + if (ackMessage == nullptr) { + LOGE("[AbilitySync][SendAck] message create failed, may be memleak!"); + return -E_OUT_OF_MEMORY; + } + int errCode = ackMessage->SetCopiedObject<>(ackPacket); + if (errCode != E_OK) { + LOGE("[AbilitySync][SendAck] SetCopiedObject failed, err %d", errCode); + delete ackMessage; + ackMessage = nullptr; + return errCode; + } + (!isAckNotify) ? ackMessage->SetMessageType(TYPE_RESPONSE) : ackMessage->SetMessageType(TYPE_NOTIFY); + ackMessage->SetTarget(deviceId_); + ackMessage->SetSessionId(inMsg->GetSessionId()); + ackMessage->SetSequenceId(inMsg->GetSequenceId()); + errCode = communicator_->SendMessage(deviceId_, ackMessage, false, SEND_TIME_OUT); + if (errCode != E_OK) { + LOGE("[AbilitySync][SendAck] SendPacket failed, err %d", errCode); + delete ackMessage; + ackMessage = nullptr; + } + return errCode; +} + +SyncOpinion AbilitySync::MakeKvSyncOpnion(const AbilitySyncRequestPacket *packet, const std::string &remoteSchema) const +{ + uint8_t remoteSchemaType = packet->GetSchemaType(); + SchemaObject localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); + SyncOpinion localSyncOpinion = SchemaObject::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); + return localSyncOpinion; +} + +RelationalSyncOpinion AbilitySync::MakeRelationSyncOpnion(const AbilitySyncRequestPacket *packet, + const std::string &remoteSchema) const +{ + uint8_t remoteSchemaType = packet->GetSchemaType(); + RelationalSchemaObject localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); + return RelationalSchemaObject::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); +} + +void AbilitySync::HandleKvAckSchemaParam(const AbilitySyncAckPacket *recvPacket, + ISyncTaskContext *context, AbilitySyncAckPacket &sendPacket) const +{ + std::string remoteSchema = recvPacket->GetSchema(); + uint8_t remoteSchemaType = recvPacket->GetSchemaType(); + bool permitSync = static_cast(recvPacket->GetPermitSync()); + bool requirePeerConvert = static_cast(recvPacket->GetRequirePeerConvert()); + SyncOpinion remoteOpinion = {permitSync, requirePeerConvert, true}; + SchemaObject localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); + SyncOpinion syncOpinion = SchemaObject::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); + SyncStrategy localStrategy = SchemaObject::ConcludeSyncStrategy(syncOpinion, remoteOpinion); + SetAbilityAckSyncOpinionInfo(sendPacket, syncOpinion); + (static_cast(context))->SetSyncStrategy(localStrategy); +} + +int AbilitySync::HandleRelationAckSchemaParam(const AbilitySyncAckPacket *recvPacket, + AbilitySyncAckPacket &sendPacket, ISyncTaskContext *context, bool sendOpinion) const +{ + std::string remoteSchema = recvPacket->GetSchema(); + uint8_t remoteSchemaType = recvPacket->GetSchemaType(); + auto localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); + auto localOpinion = RelationalSchemaObject::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); + auto localStrategy = RelationalSchemaObject::ConcludeSyncStrategy(localOpinion, + recvPacket->GetRelationalSyncOpinion()); + (static_cast(context))->SetRelationalSyncStrategy(localStrategy); + int errCode = (static_cast(storageInterface_))-> + CreateDistributedDeviceTable(context->GetDeviceId(), localStrategy); + if (errCode != E_OK) { + LOGE("[AbilitySync][AckRecv] create distributed device table failed,errCode=%d", errCode); + } + if (sendOpinion) { + sendPacket.SetRelationalSyncOpinion(localOpinion); + } + return errCode; +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h index a76f8dc62..e1fa1ad0c 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h @@ -117,6 +117,10 @@ public: void SetDbAbility(const DbAbility &dbAbility); + void SetRelationalSyncOpinion(const RelationalSyncOpinion &relationalSyncOpinion); + + RelationalSyncOpinion GetRelationalSyncOpinion() const; + private: uint32_t protocolVersion_; uint32_t softwareVersion_; @@ -129,6 +133,7 @@ private: uint32_t requirePeerConvert_; uint64_t dbCreateTime_; DbAbility dbAbility_; + RelationalSyncOpinion relationalSyncOpinion_; }; class AbilitySync { @@ -164,13 +169,6 @@ public: static int DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); // register to communicator private: -#ifdef RELATIONAL_STORE - int SendAck(const Message *inMsg, const ISchema &schemaObj, int ackCode, SyncOpinion localOpinion, - bool isAckNotify = false); -#else - int SendAck(const Message *inMsg, const SchemaObject &schemaObj, int ackCode, SyncOpinion localOpinion, - bool isAckNotify = false); -#endif static int RequestPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); @@ -191,36 +189,51 @@ private: bool SecLabelCheck(const AbilitySyncRequestPacket *packet) const; - SyncOpinion HandleVersionV3RequestParam(const AbilitySyncRequestPacket *packet, ISyncTaskContext *context, + void HandleVersionV3RequestParam(const AbilitySyncRequestPacket *packet, ISyncTaskContext *context, const std::string &remoteSchema) const; void HandleVersionV3AckSecOptionParam(const AbilitySyncAckPacket *packet, ISyncTaskContext *context) const; - SyncOpinion HandleVersionV3AckSchemaParam(const AbilitySyncAckPacket *packet, const std::string &remoteSchema, - ISyncTaskContext *context) const; + int HandleVersionV3AckSchemaParam(const AbilitySyncAckPacket *recvPacket, + AbilitySyncAckPacket &sendPacket, ISyncTaskContext *context, bool sendOpinion) const; + + void HandleKvAckSchemaParam(const AbilitySyncAckPacket *recvPacket, + ISyncTaskContext *context, AbilitySyncAckPacket &sendPacket) const; + + int HandleRelationAckSchemaParam(const AbilitySyncAckPacket *recvPacket, + AbilitySyncAckPacket &sendPacket, ISyncTaskContext *context, bool sendOpinion) const; void GetPacketSecOption(SecurityOption &option) const; int SetAbilityRequestBodyInfo(AbilitySyncRequestPacket &packet, uint16_t remoteCommunicatorVersion) const; -#ifdef RELATIONAL_STORE - int SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const ISchema &schemaObj, int ackCode, - SyncOpinion localOpinion, bool isAckNotify); -#else - int SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const SchemaObject &schemaObj, int ackCode, - SyncOpinion localOpinion, bool isAckNotify); -#endif + int SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, int ackCode, bool isAckNotify) const; + + void SetAbilityAckSchemaInfo(AbilitySyncAckPacket &ackPacket, const ISchema &schemaObj) const; + + void SetAbilityAckSyncOpinionInfo(AbilitySyncAckPacket &ackPacket, SyncOpinion localOpinion) const; int GetDbAbilityInfo(DbAbility &dbAbility) const; int AckMsgCheck(const Message *message, ISyncTaskContext *context) const; bool IsSingleKvVer() const; -#ifdef RELATIONAL_STORE + bool IsSingleRelationalVer() const; -#endif - int SendAck(const Message *message, SyncOpinion &localSyncOpinion, int ackCode, bool isAckNotify); + + int SendAck(const Message *message, int ackCode, bool isAckNotify, AbilitySyncAckPacket &ackPacket); + + int SendAckWithEmptySchema(const Message *message, int ackCode, bool isAckNotify); + + int SendAck(const Message *message, const AbilitySyncAckPacket &ackPacket, bool isAckNotify); + + int HandleRequestRecv(const Message *message, ISyncTaskContext *context, bool isCompatible); + + SyncOpinion MakeKvSyncOpnion(const AbilitySyncRequestPacket *packet, const std::string &remoteSchema) const; + + RelationalSyncOpinion MakeRelationSyncOpnion(const AbilitySyncRequestPacket *packet, + const std::string &remoteSchema) const; ICommunicator *communicator_; ISyncInterface *storageInterface_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/isync_engine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_engine.h index f7db71ff9..c073be6a1 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/isync_engine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_engine.h @@ -76,6 +76,8 @@ public: // Check if the Sync Engine is active, some times synchronization is not allowed virtual bool IsEngineActive() const = 0; + virtual void ResetAbilitySync() = 0; + protected: virtual ~ISyncEngine() {}; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index 214095e88..4a6d2c545 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -1687,7 +1687,12 @@ bool SingleVerDataSync::CheckPermitReceiveData(const SingleVerSyncTaskContext *c int SingleVerDataSync::CheckSchemaStrategy(SingleVerSyncTaskContext *context, const Message *message) { - SyncStrategy localStrategy = context->GetSyncStrategy(); + auto *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + auto query = packet->GetQuery(); + SyncStrategy localStrategy = context->GetSyncStrategy(query); if (!context->GetIsSchemaSync()) { LOGE("[DataSync][CheckSchemaStrategy] isSchemaSync=%d check failed", context->GetIsSchemaSync()); (void)SendDataAck(context, message, -E_NEED_ABILITY_SYNC, 0); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.cpp index 842fb0b84..ab67a2223 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.cpp @@ -33,4 +33,18 @@ std::string SingleVerKvSyncTaskContext::GetDeleteSyncId() const { return GetDeviceId(); } + +void SingleVerKvSyncTaskContext::SetSyncStrategy(SyncStrategy strategy) +{ + syncStrategy_.permitSync = strategy.permitSync; + syncStrategy_.convertOnSend = strategy.convertOnSend; + syncStrategy_.convertOnReceive = strategy.convertOnReceive; + syncStrategy_.checkOnReceive = strategy.checkOnReceive; +} + +SyncStrategy SingleVerKvSyncTaskContext::GetSyncStrategy(QuerySyncObject &querySyncObject) const +{ + return syncStrategy_; +} + } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.h index b30da0f23..50e3ca80f 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_sync_task_context.h @@ -29,8 +29,12 @@ public: std::string GetQuerySyncId() const override; std::string GetDeleteSyncId() const override; + void SetSyncStrategy(SyncStrategy strategy); + SyncStrategy GetSyncStrategy(QuerySyncObject &querySyncObject) const override; protected: ~SingleVerKvSyncTaskContext() override; + + SyncStrategy syncStrategy_; }; } #endif // SINGLE_VER_KV_SYNC_TASK_CONTEXT_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.cpp index 907db1520..b006c4105 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.cpp @@ -49,5 +49,15 @@ void SingleVerRelationalSyncTaskContext::CopyTargetData(const ISyncTarget *targe querySyncId_ = query_.GetRelationTableName() + query_.GetIdentify(); // save as deviceId + tableName + queryId deleteSyncId_ = GetDeviceId() + query_.GetRelationTableName(); // save as deviceId + tableName } + +void SingleVerRelationalSyncTaskContext::SetRelationalSyncStrategy(RelationalSyncStrategy strategy) +{ + relationalSyncStrategy_ = strategy; +} + +SyncStrategy SingleVerRelationalSyncTaskContext::GetSyncStrategy(QuerySyncObject &querySyncObject) const +{ + return relationalSyncStrategy_.GetTableStrategy(querySyncObject.GetRelationTableName()); +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.h index b5f0d338d..c1024fcb9 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_sync_task_context.h @@ -31,12 +31,17 @@ public: std::string GetQuerySyncId() const override; std::string GetDeleteSyncId() const override; + void SetRelationalSyncStrategy(RelationalSyncStrategy strategy); + SyncStrategy GetSyncStrategy(QuerySyncObject &querySyncObject) const override; protected: ~SingleVerRelationalSyncTaskContext() override; void CopyTargetData(const ISyncTarget *target, const TaskParam &taskParam) override; std::string querySyncId_; std::string deleteSyncId_; + + // for relational syncStrategy + RelationalSyncStrategy relationalSyncStrategy_; }; } #endif // RELATIONAL_STORE diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp index 7e951f458..f489f8c69 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp @@ -19,6 +19,17 @@ #include "single_ver_sync_engine.h" namespace DistributedDB { +int SingleVerRelationalSyncer::Initialize(ISyncInterface *syncInterface) +{ + int errCode = SingleVerSyncer::Initialize(syncInterface); + if (errCode != E_OK) { + return errCode; + } + auto callback = std::bind(&SingleVerRelationalSyncer::SchemaChangeCallback, this); + return static_cast(syncInterface)-> + RegisterSchemaChangedCallback(callback); +} + int SingleVerRelationalSyncer::Sync(const SyncParma ¶m) { if (param.mode == SYNC_MODE_PUSH_PULL) { @@ -60,7 +71,7 @@ int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint for (const QuerySyncObject &table : tablesQuery) { uint32_t subSyncId = GenerateSyncId(); LOGI("[SingleVerRelationalSyncer] SubSyncId %d create by SyncId %d, tableName = %s", - subSyncId, syncId, table.GetRelationTableName().c_str()); + subSyncId, syncId, STR_MASK(DBCommon::TransferHashString(table.GetRelationTableName()))); subParam.syncQuery = table; subParam.onComplete = std::bind(&SingleVerRelationalSyncer::DoOnSubSyncComplete, this, subSyncId, syncId, param, std::placeholders::_1); @@ -141,6 +152,13 @@ void SingleVerRelationalSyncer::LocalDataChanged(int notifyEvent) { } +void SingleVerRelationalSyncer::SchemaChangeCallback() +{ + if (syncEngine_ != nullptr) { + syncEngine_->ResetAbilitySync(); + } +} + int SingleVerRelationalSyncer::SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, const std::vector &devices) const { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h index f4696e2c7..bf6bd992b 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h @@ -22,6 +22,8 @@ public: SingleVerRelationalSyncer() = default; ~SingleVerRelationalSyncer() override = default; + int Initialize(ISyncInterface *syncInterface) override; + // Sync function. use SyncParma to reduce paramter. int Sync(const SyncParma ¶m) override; @@ -47,6 +49,8 @@ private: void DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, const SyncParma ¶m, const std::map &devicesMap); + void SchemaChangeCallback(); + mutable std::mutex syncMapLock_; std::map> fullSyncIdMap_; std::map>> resMap_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp index a59713da0..0a465d60e 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp @@ -836,6 +836,7 @@ int SingleVerSyncStateMachine::GetSyncOperationStatus(int errCode) const { -E_NOT_SUPPORT, SyncOperation::OP_NOT_SUPPORT }, { -E_INTERCEPT_DATA_FAIL, SyncOperation::OP_INTERCEPT_DATA_FAIL }, { -E_MAX_LIMITS, SyncOperation::OP_MAX_LIMITS }, + { -E_SCHEMA_CHANGE, SyncOperation::OP_SCHEMA_CHANGED }, { -E_NOT_REGISTER, SyncOperation::OP_NOT_SUPPORT }, }; auto iter = statusMap.find(errCode); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp index c74ac8aa8..2de9e337d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp @@ -351,20 +351,6 @@ bool SingleVerSyncTaskContext::GetSendPermitCheck() const { return isSendPermitChecked_; } - -void SingleVerSyncTaskContext::SetSyncStrategy(SyncStrategy strategy) -{ - syncStrategy_.permitSync = strategy.permitSync; - syncStrategy_.convertOnSend = strategy.convertOnSend; - syncStrategy_.convertOnReceive = strategy.convertOnReceive; - syncStrategy_.checkOnReceive = strategy.checkOnReceive; -} - -SyncStrategy SingleVerSyncTaskContext::GetSyncStrategy() const -{ - return syncStrategy_; -} - void SingleVerSyncTaskContext::SetIsSchemaSync(bool isSchemaSync) { isSchemaSync_ = isSchemaSync; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h index 754896de9..5e9afe2b1 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h @@ -109,9 +109,7 @@ public: bool GetSendPermitCheck() const; - void SetSyncStrategy(SyncStrategy strategy); - - SyncStrategy GetSyncStrategy() const; + virtual SyncStrategy GetSyncStrategy(QuerySyncObject &querySyncObject) const = 0; void SetIsSchemaSync(bool isChecked); @@ -162,7 +160,6 @@ private: SecurityOption remoteSecOption_ = {0, 0}; // remote targe can handle secOption data or not. bool isReceivcPermitChecked_ = false; bool isSendPermitChecked_ = false; - SyncStrategy syncStrategy_; bool isSchemaSync_ = false; // is receive waterMark err, peerWaterMark bigger than remote localWaterMark diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp index abd479b87..700bdb0a8 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp @@ -969,4 +969,19 @@ bool SyncEngine::IsEngineActive() const { return isActive_; } + +void SyncEngine::ResetAbilitySync() +{ + std::lock_guard lock(contextMapLock_); + for (auto &enrty : syncTaskContextMap_) { + auto context = enrty.second; + if (context->IsKilled()) { + continue; + } + // IncRef for SyncEngine to make sure context is valid, to avoid a big lock + RefObject::IncObjRef(context); + context->SetIsNeedResetAbilitySync(true); + RefObject::DecObjRef(context); + } +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h index 40e36e811..ca4d176d6 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h @@ -106,6 +106,8 @@ public: bool IsEngineActive() const override; + void ResetAbilitySync() override; + protected: // Create a context virtual ISyncTaskContext *CreateSyncTaskContext() = 0; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp index 1a8afd21b..04330caca 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp @@ -286,6 +286,7 @@ const std::map &SyncOperation::DBStatusTransMap() { static_cast(OP_NOT_SUPPORT), NOT_SUPPORT }, { static_cast(OP_INTERCEPT_DATA_FAIL), INTERCEPT_DATA_FAIL }, { static_cast(OP_MAX_LIMITS), OVER_MAX_LIMITS }, + { static_cast(OP_SCHEMA_CHANGED), RELATIONAL_SCHEMA_CHANGED }, { static_cast(OP_INVALID_ARGS), INVALID_ARGS }, }; return transMap; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h index 430f23f77..46510a772 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h @@ -52,6 +52,7 @@ public: OP_NOT_SUPPORT, OP_INTERCEPT_DATA_FAIL, OP_MAX_LIMITS, + OP_SCHEMA_CHANGED, OP_INVALID_ARGS }; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_value_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_value_test.cpp new file mode 100644 index 000000000..eb2c79b4e --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_value_test.cpp @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "distributeddb_tools_unit_test.h" +#include "data_value.h" + +using namespace testing::ext; +using namespace DistributedDB; + +namespace { +const int ONE_HUNDERED = 100; +const char DEFAULT_CHAR = 'D'; +const std::string DEFAULT_TEXT = "This is a text"; + +void DataValueDefaultNullCheck(DataValue &dataValue) +{ + /** + * @tc.steps: step1. create a dataValue + * @tc.expected: dataValue type is null. + */ + dataValue.ResetValue(); + EXPECT_EQ(dataValue.GetType(), StorageType::STORAGE_TYPE_NULL); +} + +void DataValueBoolCheck(DataValue &dataValue) +{ + /** + * @tc.steps: step1. create a dataValue and set true + * @tc.expected: dataValue type is bool. + */ + bool targetVal = true; + dataValue = targetVal; + EXPECT_EQ(dataValue.GetType(), StorageType::STORAGE_TYPE_BOOL); + /** + * @tc.steps: step2. get a value from dataValue + * @tc.expected: get ok and value is true. + */ + bool val = false; + EXPECT_EQ(dataValue.GetBool(val), E_OK); + EXPECT_EQ(val, targetVal); + /** + * @tc.steps: step3. set false and then get it from dataValue. + * @tc.expected: get ok and value is false. + */ + targetVal = false; + dataValue = targetVal; + EXPECT_EQ(dataValue.GetType(), StorageType::STORAGE_TYPE_BOOL); + EXPECT_EQ(dataValue.GetBool(val), E_OK); + EXPECT_EQ(targetVal, val); +} + +void DataValueDoubleCheck(DataValue &dataValue) +{ + /** + * @tc.steps: step1. create a dataValue and set true + * @tc.expected: dataValue type is double. + */ + double targetDoubleVal = 1.0; + dataValue = targetDoubleVal; + EXPECT_EQ(dataValue.GetType(), StorageType::STORAGE_TYPE_REAL); + /** + * @tc.steps: step2. get a double from dataValue + * @tc.expected: get ok and value is equal to targetDoubleVal. + */ + double val = 0; + EXPECT_EQ(dataValue.GetDouble(val), E_OK); + EXPECT_EQ(val, targetDoubleVal); +} + +void DataValueInt64Check(DataValue &dataValue) +{ + /** + * @tc.steps: step1. create a dataValue and set INTE64_MAX + * @tc.expected: dataValue type is int64. + */ + int64_t targetInt64Val = INT64_MAX; + dataValue = targetInt64Val; + EXPECT_EQ(dataValue.GetType(), StorageType::STORAGE_TYPE_INTEGER); + /** + * @tc.steps: step2. get a int64 from dataValue + * @tc.expected: get ok and value is equal to INTE64_MAX. + */ + int64_t val = INT64_MIN; + EXPECT_EQ(dataValue.GetInt64(val), E_OK); + EXPECT_EQ(val, targetInt64Val); +} + +void DataValueStringCheck(DataValue &dataValue) +{ + /** + * @tc.steps: step1. set a string + * @tc.expected: dataValue type is string. + */ + dataValue.SetText(DEFAULT_TEXT); + EXPECT_EQ(dataValue.GetType(), StorageType::STORAGE_TYPE_TEXT); + /** + * @tc.steps: step2. get a string from dataValue + * @tc.expected: get ok and string is equal to DEFAULT_TEXT. + */ + std::string val; + EXPECT_EQ(dataValue.GetText(val), E_OK); + EXPECT_EQ(val, DEFAULT_TEXT); +} + +void DataValueBlobCheck(DataValue &dataValue) +{ + /** + * @tc.steps: step1. set a blob + * @tc.expected: dataValue type is blob. + */ + Blob targetVal; + std::vector vector(ONE_HUNDERED, DEFAULT_CHAR); + targetVal.WriteBlob(reinterpret_cast(vector.data()), vector.size()); + dataValue.SetBlob(targetVal); + EXPECT_EQ(dataValue.GetType(), StorageType::STORAGE_TYPE_BLOB); + /** + * @tc.steps: step2. get a blob from dataValue + * @tc.expected: get ok and value is equal to DEFAULT_TEXT. + */ + Blob val; + EXPECT_EQ(dataValue.GetBlob(val), E_OK); + for (int i = 0; i < ONE_HUNDERED; i++) { + EXPECT_EQ(targetVal.GetData()[i], val.GetData()[i]); + } +} + +const std::vector checkFuncList = { + &DataValueDefaultNullCheck, &DataValueBoolCheck, &DataValueInt64Check, + &DataValueDoubleCheck, &DataValueStringCheck, &DataValueBlobCheck +}; + +class DistribubtedDBInterfacesDataValueTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistribubtedDBInterfacesDataValueTest::SetUpTestCase(void) +{ +} + +void DistribubtedDBInterfacesDataValueTest::TearDownTestCase(void) +{ +} + +void DistribubtedDBInterfacesDataValueTest::SetUp(void) +{ +} + +void DistribubtedDBInterfacesDataValueTest::TearDown(void) +{ +} + +/** + * @tc.name: DataValueCheck001 + * @tc.desc: To test dataValue work correctly when data is nullptr, bool, string, double, int64, uint8_t*. + * @tc.type: Func + * @tc.require: + * @tc.author: zhangqiquan + */ +HWTEST_F(DistribubtedDBInterfacesDataValueTest, DataValueCheck001, TestSize.Level1) +{ + for (const auto &func : checkFuncList) { + DataValue dataValue; + func(dataValue); + } +} + +/** + * @tc.name: DataValueCheck002 + * @tc.desc: To test dataValue work correctly when different type overwrite into dataValue. + * @tc.type: Func + * @tc.require: + * @tc.author: zhangqiquan + */ +HWTEST_F(DistribubtedDBInterfacesDataValueTest, DataValueCheck002, TestSize.Level1) +{ + for (uint32_t lastWriteIndex = 0; lastWriteIndex < checkFuncList.size(); lastWriteIndex++) { + DataValue dataValue; + for (uint32_t i = 0; i < checkFuncList.size(); i++) { + uint32_t index = (lastWriteIndex + i + 1) % static_cast(checkFuncList.size()); + checkFuncList[index](dataValue); + } + } +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_data_transformer_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_data_transformer_test.cpp new file mode 100644 index 000000000..91609a351 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_data_transformer_test.cpp @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "data_transformer.h" + +using namespace testing::ext; +using namespace DistributedDB; +namespace { +const int ONE_HUNDERED = 100; +const char DEFAULT_CHAR = 'D'; +const std::string DEFAULT_TEXT = "This is a text"; +const std::vector DEFAULT_BLOB(ONE_HUNDERED, DEFAULT_CHAR); +void SetNull(DataValue &dataValue) +{ + dataValue.ResetValue(); +} + +void SetBool(DataValue &dataValue) +{ + dataValue = false; +} + +void SetInt64(DataValue &dataValue) +{ + dataValue = INT64_MAX; +} + +void SetDouble(DataValue &dataValue) +{ + dataValue = 1.0; +} + +void SetText(DataValue &dataValue) +{ + dataValue.SetText(DEFAULT_TEXT); +} + +void SetBlob(DataValue &dataValue) +{ + Blob blob; + blob.WriteBlob(DEFAULT_BLOB.data(), DEFAULT_BLOB.size()); + dataValue.SetBlob(blob); +} + +std::map typeMapFunction = { + {StorageType::STORAGE_TYPE_NULL, &SetNull}, + {StorageType::STORAGE_TYPE_BOOL, &SetBool}, + {StorageType::STORAGE_TYPE_INTEGER, &SetInt64}, + {StorageType::STORAGE_TYPE_REAL, &SetDouble}, + {StorageType::STORAGE_TYPE_TEXT, &SetText}, + {StorageType::STORAGE_TYPE_BLOB, &SetBlob} +}; + +void GenerateRowData(const std::vector &fieldInfoList, RowData &rowData) +{ + for (auto &item: fieldInfoList) { + DataValue dataValue; + StorageType type = StorageType::STORAGE_TYPE_NULL; + if (typeMapFunction.find(item.GetStorageType()) != typeMapFunction.end()) { + type = item.GetStorageType(); + } + typeMapFunction[type](dataValue); + rowData.push_back(std::move(dataValue)); + } +} + +void GenerateTableDataWithLog(const std::vector &fieldInfoList, TableDataWithLog &tableDataWithLog) +{ + tableDataWithLog.tableName = DEFAULT_TEXT; + for (int i = 0; i < ONE_HUNDERED; i++) { + RowDataWithLog rowDataWithLog; + GenerateRowData(fieldInfoList, rowDataWithLog.rowData); + LogInfo logInfo; + // choose first element as primary key + logInfo.dataKey = i; + tableDataWithLog.dataList.push_back(std::move(rowDataWithLog)); + } +} + +bool Equal(const LogInfo &origin, const LogInfo &target) +{ + if (origin.dataKey != target.dataKey) { + return false; + } + if (origin.device != target.device) { + return false; + } + if (origin.flag != target.flag) { + return false; + } + if (origin.hashKey != target.hashKey) { + return false; + } + if (origin.originDev != target.originDev) { + return false; + } + if (origin.timestamp != target.timestamp) { + return false; + } + if (origin.wTimeStamp != target.wTimeStamp) { + return false; + } + return true; +} + +bool Equal(const RowDataWithLog &origin, const OptRowDataWithLog &target) +{ + if (!Equal(origin.logInfo, target.logInfo)) { + return false; + } + if (origin.rowData.size() != target.optionalData.size()) { + return false; + } + for (uint32_t i = 0; i < origin.rowData.size(); i++) { + const auto &originData = origin.rowData[i]; + const auto &targetData = target.optionalData[i]; + if (!targetData.has_value()) { + return false; + } + if (originData != targetData.value()) { + LOGD("VALUE NOT EQUAL!"); + return false; + } + } + return true; +} + +bool Equal(TableDataWithLog origin, OptTableDataWithLog target) +{ + if (origin.dataList.size() != target.dataList.size()) { + return false; + } + for (uint32_t i = 0; i < origin.dataList.size(); i++) { + RowDataWithLog originData = origin.dataList[i]; + OptRowDataWithLog targetData = target.dataList[i]; + if (!Equal(originData, targetData)) { + return false; + } + } + return true; +} + +void GenerateDiffFieldInfo(std::vector &sendFieldInfoList, std::vector &receiveFieldInfoList) +{ + const int columnCounts = 6; // 6 columns + const char sendColumn = 'A'; // sendColumn start with 'A' + const char receiveColumn = 'B'; // receiveColumn start with 'B' + // we generate 2 diff schema here (A, B, C, D, E, F) and (B, C, D, E, F, G) + + for (uint32_t i = 0; i < columnCounts; i++) { + FieldInfo sendFieldInfo; + sendFieldInfo.SetFieldName(std::string(1, static_cast(sendColumn + i))); + sendFieldInfo.SetStorageType(StorageType::STORAGE_TYPE_INTEGER); + sendFieldInfoList.push_back(sendFieldInfo); + FieldInfo receiveFieldInfo; + receiveFieldInfo.SetFieldName(std::string(1, static_cast(receiveColumn + i))); + receiveFieldInfo.SetStorageType(StorageType::STORAGE_TYPE_INTEGER); + receiveFieldInfoList.push_back(receiveFieldInfo); + } +} +} + +class DistributedDBDataTransformerTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBDataTransformerTest::SetUpTestCase(void) +{ +} + +void DistributedDBDataTransformerTest::TearDownTestCase(void) +{ +} + +void DistributedDBDataTransformerTest::SetUp(void) +{ +} + +void DistributedDBDataTransformerTest::TearDown(void) +{ +} + +/** + * @tc.name: DataTransformerCheck001 + * @tc.desc: To test transformer work correctly when data contains nullptr, bool, string, double, int64, uint8_t*. + * @tc.type: Func + * @tc.require: + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBDataTransformerTest, DataTransformerCheck001, TestSize.Level1) +{ + /** + * @tc.steps: step1. generate the fieldInfoList which contains nullptr, bool, string, double, int64, uint8_t*. + */ + std::vector fieldInfoList; + int count = 0; + for (const auto &item : typeMapFunction) { + FieldInfo fieldInfo; + fieldInfo.SetStorageType(item.first); + fieldInfo.SetFieldName(std::to_string(count++)); + fieldInfoList.push_back(fieldInfo); + } + + /** + * @tc.steps: step2. generate a originData by fieldInfoLiist. + */ + TableDataWithLog originData; + GenerateTableDataWithLog(fieldInfoList, originData); + + /** + * @tc.steps: step3. transform originData to KV data and transform back to relationData. + * @tc.expected: get ok and value has no change + */ + std::vector dataItemList; + EXPECT_EQ(DataTransformer::TransformTableData(originData, fieldInfoList, dataItemList), E_OK); + + OptTableDataWithLog targetData; + EXPECT_EQ(DataTransformer::TransformDataItem(dataItemList, fieldInfoList, fieldInfoList, targetData), E_OK); + + EXPECT_TRUE(Equal(originData, targetData)); +} + +/** + * @tc.name: DataTransformerCheck002 + * @tc.desc: To test transformer work correctly when table is different between send and receive. + * @tc.type: Func + * @tc.require: + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBDataTransformerTest, DataTransformerCheck002, TestSize.Level1) +{ + /** + * @tc.steps: step1. generate the fieldInfoList which contains nullptr, bool, string, double, int64, uint8_t*. + */ + std::vector sendFieldInfoList; + std::vector receiveFieldInfoList; + GenerateDiffFieldInfo(sendFieldInfoList, receiveFieldInfoList); + + /** + * @tc.steps: step2. generate a originData by fieldInfoList. + */ + TableDataWithLog originData; + GenerateTableDataWithLog(sendFieldInfoList, originData); + + /** + * @tc.steps: step3. get a value from dataValue. + * @tc.expected: get ok and value is true + */ + std::vector dataItemList; + EXPECT_EQ(DataTransformer::TransformTableData(originData, sendFieldInfoList, dataItemList), E_OK); + + OptTableDataWithLog targetData; + EXPECT_EQ(DataTransformer::TransformDataItem(dataItemList, sendFieldInfoList, receiveFieldInfoList, + targetData), E_OK); + + DataValue target; + target = INT64_MAX; + EXPECT_EQ(targetData.dataList.size(), dataItemList.size()); + for (const auto &optDataWithLog : targetData.dataList) { + for (uint32_t i = 0; i < optDataWithLog.optionalData.size(); i++) { + const std::optional &optData = optDataWithLog.optionalData[i]; + if (i == optDataWithLog.optionalData.size() - 1) { + EXPECT_TRUE(!optData.has_value()); + } else { + EXPECT_TRUE(optData.has_value()); + EXPECT_EQ(optData.value(), target); + } + } + } +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index 06eec91d2..3d80e4dc5 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -35,24 +35,37 @@ using namespace DistributedDBUnitTest; namespace { const std::string DEVICE_B = "deviceB"; const std::string g_tableName = "TEST_TABLE"; - const std::string g_syncTableName = "naturalbase_rdb_aux_" + - g_tableName + "_" + DBCommon::TransferStringToHex(DBCommon::TransferHashString(DEVICE_B)); + + const int ONE_HUNDERED = 100; + const char DEFAULT_CHAR = 'D'; + const std::string DEFAULT_TEXT = "This is a text"; + const std::vector DEFAULT_BLOB(ONE_HUNDERED, DEFAULT_CHAR); RelationalStoreManager g_mgr(APP_ID, USER_ID); std::string g_testDir; std::string g_dbDir; std::string g_id; + std::vector g_storageType = { + StorageType::STORAGE_TYPE_INTEGER, StorageType::STORAGE_TYPE_REAL, + StorageType::STORAGE_TYPE_TEXT, StorageType::STORAGE_TYPE_BLOB + }; DistributedDBToolsUnitTest g_tool; - RelationalStoreDelegate* g_kvDelegatePtr = nullptr; + RelationalStoreDelegate* g_rdbDelegatePtr = nullptr; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; RelationalVirtualDevice *g_deviceB = nullptr; std::vector g_fieldInfoList; + std::string GetDeviceTableName(const std::string &tableName) + { + return "naturalbase_rdb_aux_" + + tableName + "_" + DBCommon::TransferStringToHex(DBCommon::TransferHashString(DEVICE_B)); + } + void OpenStore() { - RelationalStoreDelegate::Option option; - g_mgr.OpenStore(g_dbDir, STORE_ID_1, option, g_kvDelegatePtr); - ASSERT_TRUE(g_kvDelegatePtr != nullptr); + RelationalStoreDelegate::Option option = {}; + g_mgr.OpenStore(g_dbDir, STORE_ID_1, option, g_rdbDelegatePtr); + ASSERT_TRUE(g_rdbDelegatePtr != nullptr); } int GetDB(sqlite3 *&db) @@ -69,22 +82,70 @@ namespace { return rc; } - int CreateTable(sqlite3 *db) + std::string GetType(StorageType type) { - char *pErrMsg = nullptr; + static std::map typeMap = { + {StorageType::STORAGE_TYPE_INTEGER, "INT"}, + {StorageType::STORAGE_TYPE_BOOL, "BOOL"}, + {StorageType::STORAGE_TYPE_REAL, "DOUBLE"}, + {StorageType::STORAGE_TYPE_TEXT, "TEXT"}, + {StorageType::STORAGE_TYPE_BLOB, "BLOB"} + }; + if (typeMap.find(type) == typeMap.end()) { + type = StorageType::STORAGE_TYPE_INTEGER; + } + return typeMap[type]; + } - const char *sql1 = "CREATE TABLE TEST_TABLE(" \ - "ID INT PRIMARY KEY NOT NULL," \ - "NAME TEXT ," \ - "AGE INT);"; - return sqlite3_exec(db, sql1, nullptr, nullptr, &pErrMsg); + int DropTable(sqlite3 *db, const std::string &tableName) + { + std::string sql = "DROP TABLE " + tableName + ";"; + return sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr); } - int PrepareInsert(sqlite3 *db, sqlite3_stmt *&statement) + int CreateTable(sqlite3 *db, std::vector &fieldInfoList, const std::string &tableName) { - const char *sql = "INSERT OR REPLACE INTO TEST_TABLE (ID,NAME,AGE) " \ - "VALUES (?, ?, ?);"; - return sqlite3_prepare_v2(db, sql, -1, &statement, nullptr); + std::string sql = "CREATE TABLE " + tableName + "("; + int index = 0; + for (const auto &field : fieldInfoList) { + if (index != 0) { + sql += ","; + } + sql += field.GetFieldName() + " "; + std::string type = GetType(field.GetStorageType()); + sql += type + " "; + if (index == 0) { + sql += "PRIMARY KEY NOT NULL "; + } + index++; + } + sql += ");"; + int rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr); + return rc; + } + + int PrepareInsert(sqlite3 *db, sqlite3_stmt *&statement, + std::vector fieldInfoList, const std::string &tableName) + { + std::string sql = "INSERT OR REPLACE INTO " + tableName + "("; + int index = 0; + for (const auto &fieldInfo : fieldInfoList) { + if (index != 0) { + sql += ","; + } + sql += fieldInfo.GetFieldName(); + index++; + } + sql += ") VALUES ("; + while(index > 0) { + sql += "?"; + if (index != 1) { + sql += ", "; + } + index--; + } + sql += ");"; + return sqlite3_prepare_v2(db, sql.c_str(), -1, &statement, nullptr); } int SimulateCommitData(sqlite3 *db, sqlite3_stmt *&statement) @@ -97,44 +158,131 @@ namespace { return rc; } - void InsertValue(sqlite3 *db, RowData &rowData) + void BindValue(const DataValue &item, sqlite3_stmt *stmt, int col) + { + switch(item.GetType()) { + case StorageType::STORAGE_TYPE_BOOL: { + bool boolData = false; + (void)item.GetBool(boolData); + EXPECT_EQ(sqlite3_bind_int(stmt, col, boolData), SQLITE_OK); + break; + } + + case StorageType::STORAGE_TYPE_INTEGER: { + int64_t intData = 0; + (void)item.GetInt64(intData); + EXPECT_EQ(sqlite3_bind_int64(stmt, col, intData), SQLITE_OK); + break; + } + + case StorageType::STORAGE_TYPE_REAL: { + double doubleData = 0; + (void)item.GetDouble(doubleData); + EXPECT_EQ(sqlite3_bind_double(stmt, col, doubleData), SQLITE_OK); + break; + } + + case StorageType::STORAGE_TYPE_TEXT: { + std::string strData; + (void)item.GetText(strData); + EXPECT_EQ(SQLiteUtils::BindTextToStatement(stmt, col, strData), E_OK); + break; + } + + case StorageType::STORAGE_TYPE_BLOB: { + Blob blob; + (void)item.GetBlob(blob); + std::vector blobData(blob.GetData(), blob.GetData() + blob.GetSize()); + EXPECT_EQ(SQLiteUtils::BindBlobToStatement(stmt, col, blobData, true), E_OK); + break; + } + + case StorageType::STORAGE_TYPE_NULL: { + EXPECT_EQ(SQLiteUtils::MapSQLiteErrno(sqlite3_bind_null(stmt, col)), E_OK); + break; + } + + default: + break; + } + } + + void InsertValue(sqlite3 *db, std::map &dataMap, + std::vector &fieldInfoList, const std::string &tableName) { sqlite3_stmt *stmt = nullptr; - EXPECT_EQ(PrepareInsert(db, stmt), SQLITE_OK); - for (int i = 0; i < static_cast(rowData.size()); ++i) { - const auto &item = rowData[i]; + EXPECT_EQ(PrepareInsert(db, stmt, fieldInfoList, tableName), SQLITE_OK); + for (int i = 0; i < (int)fieldInfoList.size(); ++i) { + const auto &fieldName = fieldInfoList[i].GetFieldName(); + ASSERT_TRUE(dataMap.find(fieldName) != dataMap.end()); + const auto &item = dataMap[fieldName]; const int index = i + 1; - int errCode; - if (item.GetType() == StorageType::STORAGE_TYPE_INTEGER) { - int64_t val; - (void)item.GetInt64(val); - errCode = SQLiteUtils::BindInt64ToStatement(stmt, index, val); - } else if (item.GetType() == StorageType::STORAGE_TYPE_TEXT) { - std::string val; - (void)item.GetText(val); - errCode = SQLiteUtils::BindTextToStatement(stmt, index, val); - } - EXPECT_EQ(errCode, E_OK); + BindValue(item, stmt, index); } EXPECT_EQ(SimulateCommitData(db, stmt), SQLITE_DONE); sqlite3_finalize(stmt); } - void GenerateValue(RowData &rowData, std::map &dataMap) + void InsertValue(sqlite3 *db, std::map &dataMap) + { + InsertValue(db, dataMap, g_fieldInfoList, g_tableName); + } + + void SetNull(DataValue &dataValue) + { + dataValue.ResetValue(); + } + + void SetBool(DataValue &dataValue) + { + dataValue = false; + } + + void SetInt64(DataValue &dataValue) + { + dataValue = INT64_MAX; + } + + void SetDouble(DataValue &dataValue) + { + dataValue = 1.0; + } + + void SetText(DataValue &dataValue) + { + dataValue.SetText(DEFAULT_TEXT); + } + + void SetBlob(DataValue &dataValue) { - int64_t id = 0; - dataMap["ID"] = id; - rowData.push_back(dataMap["ID"]); - std::string val = "test"; - dataMap["NAME"].SetText(val); - rowData.push_back(dataMap["NAME"]); - dataMap["AGE"] = INT64_MAX; - rowData.push_back(dataMap["AGE"]); + Blob blob; + blob.WriteBlob(DEFAULT_BLOB.data(), DEFAULT_BLOB.size()); + dataValue.SetBlob(blob); } - void InsertFieldInfo() + void GenerateValue(std::map &dataMap, std::vector &fieldInfoList) { - g_fieldInfoList.clear(); + static std::map typeMapFunction = { + {StorageType::STORAGE_TYPE_NULL, &SetNull}, + {StorageType::STORAGE_TYPE_BOOL, &SetBool}, + {StorageType::STORAGE_TYPE_INTEGER, &SetInt64}, + {StorageType::STORAGE_TYPE_REAL, &SetDouble}, + {StorageType::STORAGE_TYPE_TEXT, &SetText}, + {StorageType::STORAGE_TYPE_BLOB, &SetBlob} + }; + for (auto &fieldInfo : fieldInfoList) { + DataValue dataValue; + if (typeMapFunction.find(fieldInfo.GetStorageType()) == typeMapFunction.end()) { + fieldInfo.SetStorageType(StorageType::STORAGE_TYPE_NULL); + } + typeMapFunction[fieldInfo.GetStorageType()](dataValue); + dataMap[fieldInfo.GetFieldName()] = std::move(dataValue); + } + } + + void InsertFieldInfo(std::vector &fieldInfoList) + { + fieldInfoList.clear(); FieldInfo columnFirst; columnFirst.SetFieldName("ID"); columnFirst.SetStorageType(StorageType::STORAGE_TYPE_INTEGER); @@ -144,22 +292,21 @@ namespace { FieldInfo columnThird; columnThird.SetFieldName("AGE"); columnThird.SetStorageType(StorageType::STORAGE_TYPE_INTEGER); - g_fieldInfoList.push_back(columnFirst); - g_fieldInfoList.push_back(columnSecond); - g_fieldInfoList.push_back(columnThird); - g_deviceB->SetLocalFieldInfo(g_fieldInfoList); + fieldInfoList.push_back(columnFirst); + fieldInfoList.push_back(columnSecond); + fieldInfoList.push_back(columnThird); } - void BlockSync(SyncMode syncMode, DBStatus exceptStatus) + void BlockSync(const std::string &tableName, SyncMode syncMode, DBStatus exceptStatus) { std::vector devices = {DEVICE_B}; - Query query = Query::Select(g_tableName); + Query query = Query::Select(tableName); std::map> statusMap; SyncStatusCallback callBack = [&statusMap]( const std::map> &devicesMap) { statusMap = devicesMap; }; - DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, query, callBack, true); + DBStatus callStatus = g_rdbDelegatePtr->Sync(devices, syncMode, query, callBack, true); EXPECT_EQ(callStatus, OK); for (const auto &tablesRes : statusMap) { for (const auto &tableStatus : tablesRes.second) { @@ -168,13 +315,18 @@ namespace { } } + void BlockSync(SyncMode syncMode, DBStatus exceptStatus) + { + BlockSync(g_tableName, syncMode, exceptStatus); + } + int PrepareSelect(sqlite3 *db, sqlite3_stmt *&statement, const std::string &table) { const std::string sql = "SELECT * FROM " + table; return sqlite3_prepare_v2(db, sql.c_str(), -1, &statement, nullptr); } - void GetDataValue(sqlite3_stmt *statement, int row, int col, DataValue &dataValue) + void GetDataValue(sqlite3_stmt *statement, int col, DataValue &dataValue) { int type = sqlite3_column_type(statement, col); switch (type) { @@ -201,98 +353,112 @@ namespace { case SQLITE_NULL: break; default: - LOGW("unknown type[%d] row[%llu] column[%d] ignore", type, row, col); + LOGW("unknown type[%d] column[%d] ignore", type, col); } } - void GetSyncDataStep(std::vector &dataList, sqlite3_stmt *statement) + void GetSyncDataStep(std::map &dataMap, sqlite3_stmt *statement, + std::vector fieldInfoList) { int columnCount = sqlite3_column_count(statement); - RowData rowData; for (int col = 0; col < columnCount; ++col) { DataValue dataValue; - GetDataValue(statement, static_cast(dataList.size() + 1), col, dataValue); - rowData.push_back(std::move(dataValue)); + GetDataValue(statement, col, dataValue); + dataMap[fieldInfoList[col].GetFieldName()] = std::move(dataValue); } - dataList.push_back(rowData); } - void GetSyncData(sqlite3 *db, std::vector &dataList) + void GetSyncData(sqlite3 *db, std::map &dataMap, const std::string &tableName, + std::vector fieldInfoList) { sqlite3_stmt *statement = nullptr; - EXPECT_EQ(PrepareSelect(db, statement, g_syncTableName), SQLITE_OK); + EXPECT_EQ(PrepareSelect(db, statement, GetDeviceTableName(tableName)), SQLITE_OK); while (true) { int rc = sqlite3_step(statement); if (rc != SQLITE_ROW) { LOGD("GetSyncData Exist by code[%d]", rc); break; } - GetSyncDataStep(dataList, statement); + GetSyncDataStep(dataMap, statement, fieldInfoList); } + sqlite3_finalize(statement); } - void InsertValueToDB(RowData &rowData) + void InsertValueToDB(std::map &dataMap) { sqlite3 *db = nullptr; EXPECT_EQ(GetDB(db), SQLITE_OK); - InsertValue(db, rowData); + InsertValue(db, dataMap); sqlite3_close(db); } - void PrepareEnvironment(std::map dataMap) + void PrepareBasicTable(const std::string &tableName, std::vector &fieldInfoList) { sqlite3 *db = nullptr; EXPECT_EQ(GetDB(db), SQLITE_OK); - EXPECT_EQ(CreateTable(db), SQLITE_OK); + if (fieldInfoList.empty()) { + InsertFieldInfo(fieldInfoList); + } + g_deviceB->SetLocalFieldInfo(fieldInfoList); + EXPECT_EQ(CreateTable(db, fieldInfoList, tableName), SQLITE_OK); + TableInfo tableInfo; + SQLiteUtils::AnalysisSchema(db, tableName, tableInfo); + g_deviceB->SetTableInfo(tableInfo); - EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName), OK); + EXPECT_EQ(g_rdbDelegatePtr->CreateDistributedTable(tableName), OK); sqlite3_close(db); - - InsertFieldInfo(); - RowData rowData; - GenerateValue(rowData, dataMap); - InsertValueToDB(rowData); } - void PrepareVirtualEnvironment(RowData &rowData) + void PrepareEnvironment(std::map &dataMap) { - sqlite3 *db = nullptr; - EXPECT_EQ(GetDB(db), SQLITE_OK); - EXPECT_EQ(CreateTable(db), SQLITE_OK); - EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName), OK); - - sqlite3_close(db); + PrepareBasicTable(g_tableName, g_fieldInfoList); + GenerateValue(dataMap, g_fieldInfoList); + InsertValueToDB(dataMap); + } - std::map dataMap; - GenerateValue(rowData, dataMap); + void PrepareVirtualEnvironment(std::map &dataMap, const std::string &tableName, + std::vector fieldInfoList) + { + PrepareBasicTable(tableName, fieldInfoList); + GenerateValue(dataMap, fieldInfoList); VirtualRowData virtualRowData; for (const auto &item : dataMap) { virtualRowData.objectData.PutDataValue(item.first, item.second); } virtualRowData.logInfo.timestamp = 1; - g_deviceB->PutData(g_tableName, {virtualRowData}); - InsertFieldInfo(); + g_deviceB->PutData(tableName, {virtualRowData}); + } + + void PrepareVirtualEnvironment(std::map &dataMap) + { + PrepareVirtualEnvironment(dataMap, g_tableName, g_fieldInfoList); } - void CheckData(RowData &rowData) + void CheckData(std::map &targetMap, const std::string &tableName, + std::vector fieldInfoList) { - std::vector dataList; + std::map dataMap; sqlite3 *db = nullptr; EXPECT_EQ(GetDB(db), SQLITE_OK); - GetSyncData(db, dataList); + GetSyncData(db, dataMap, tableName, fieldInfoList); sqlite3_close(db); - ASSERT_EQ(dataList.size(), static_cast(1)); - for (size_t j = 0; j < dataList[0].size(); ++j) { - EXPECT_TRUE(dataList[0][j] == rowData[j]); + for (const auto &[fieldName, dataValue] : targetMap) { + ASSERT_TRUE(dataMap.find(fieldName) != dataMap.end()); + EXPECT_TRUE(dataMap[fieldName] == dataValue); } } - void CheckVirtualData(std::map data) + void CheckData(std::map &targetMap) + { + CheckData(targetMap, g_tableName, g_fieldInfoList); + } + + void CheckVirtualData(const std::string &tableName, std::map &data) { std::vector targetData; - g_deviceB->GetAllSyncData(g_tableName, targetData); + g_deviceB->GetAllSyncData(tableName, targetData); ASSERT_EQ(targetData.size(), 1u); for (auto &[field, value] : data) { @@ -303,6 +469,61 @@ namespace { EXPECT_TRUE(target == value); } } + + void CheckVirtualData(std::map &data) + { + CheckVirtualData(g_tableName, data); + } + + void GetFieldInfo(std::vector &fieldInfoList, std::vector typeList) + { + fieldInfoList.clear(); + for (size_t index = 0; index < typeList.size(); index++) { + const auto &type = typeList[index]; + FieldInfo fieldInfo; + fieldInfo.SetFieldName("field_" + std::to_string(index)); + fieldInfo.SetColumnId(index); + fieldInfo.SetStorageType(type); + fieldInfoList.push_back(fieldInfo); + } + } + + void InsertValueToDB(std::map &dataMap, + std::vector fieldInfoList, const std::string &tableName) + { + sqlite3 *db = nullptr; + EXPECT_EQ(GetDB(db), SQLITE_OK); + InsertValue(db, dataMap, fieldInfoList, tableName); + sqlite3_close(db); + } + + void PrepareEnvironment(std::map &dataMap, const std::string &tableName, + std::vector &localFieldInfoList, std::vector &remoteFieldInfoList) + { + sqlite3 *db = nullptr; + EXPECT_EQ(GetDB(db), SQLITE_OK); + + EXPECT_EQ(CreateTable(db, remoteFieldInfoList, tableName), SQLITE_OK); + TableInfo tableInfo; + SQLiteUtils::AnalysisSchema(db, tableName, tableInfo); + g_deviceB->SetTableInfo(tableInfo); + + EXPECT_EQ(DropTable(db, tableName), SQLITE_OK); + EXPECT_EQ(CreateTable(db, localFieldInfoList, tableName), SQLITE_OK); + EXPECT_EQ(g_rdbDelegatePtr->CreateDistributedTable(tableName), OK); + + sqlite3_close(db); + + GenerateValue(dataMap, localFieldInfoList); + InsertValueToDB(dataMap, localFieldInfoList, tableName); + g_deviceB->SetLocalFieldInfo(remoteFieldInfoList); + } + + void PrepareEnvironment(std::map &dataMap, + std::vector &localFieldInfoList, std::vector &remoteFieldInfoList) + { + PrepareEnvironment(dataMap, g_tableName, localFieldInfoList, remoteFieldInfoList); + } } class DistributedDBRelationalVerP2PSyncTest : public testing::Test { @@ -320,12 +541,15 @@ void DistributedDBRelationalVerP2PSyncTest::SetUpTestCase() */ DistributedDBToolsUnitTest::TestDirInit(g_testDir); g_dbDir = g_testDir + "/test.db"; + sqlite3 *db = nullptr; + ASSERT_EQ(GetDB(db), SQLITE_OK); + sqlite3_close(db); g_communicatorAggregator = new (std::nothrow) VirtualCommunicatorAggregator(); ASSERT_TRUE(g_communicatorAggregator != nullptr); RuntimeContext::GetInstance()->SetCommunicatorAggregator(g_communicatorAggregator); - g_id = g_mgr.GetRelationalStoreIdentifier(APP_ID, USER_ID, STORE_ID_1); + g_id = g_mgr.GetRelationalStoreIdentifier(USER_ID, APP_ID, STORE_ID_1); } void DistributedDBRelationalVerP2PSyncTest::TearDownTestCase() @@ -368,10 +592,10 @@ void DistributedDBRelationalVerP2PSyncTest::TearDown(void) /** * @tc.teardown: Release device A, B, C */ - if (g_kvDelegatePtr != nullptr) { + if (g_rdbDelegatePtr != nullptr) { LOGD("CloseStore Start"); - ASSERT_EQ(g_mgr.CloseStore(g_kvDelegatePtr), OK); - g_kvDelegatePtr = nullptr; + ASSERT_EQ(g_mgr.CloseStore(g_rdbDelegatePtr), OK); + g_rdbDelegatePtr = nullptr; } if (g_deviceB != nullptr) { delete g_deviceB; @@ -379,6 +603,7 @@ void DistributedDBRelationalVerP2PSyncTest::TearDown(void) } PermissionCheckCallbackV2 nullCallback; EXPECT_EQ(RuntimeConfig::SetPermissionCheckCallback(nullCallback), OK); + EXPECT_EQ(DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir), OK); LOGD("TearDown FINISH"); } @@ -434,14 +659,11 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync003, TestSize.Level1) int64_t val = 0; - RowData rowData; - GenerateValue(rowData, dataMap); - EXPECT_EQ(rowData[rowData.size() - 1].GetInt64(val), E_OK); - rowData[rowData.size() - 1] = static_cast(1); - InsertValueToDB(rowData); + GenerateValue(dataMap, g_fieldInfoList); + dataMap["AGE"] = static_cast(1); + InsertValueToDB(dataMap); BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); - dataMap["AGE"] = static_cast(1); CheckVirtualData(dataMap); } @@ -463,7 +685,7 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync004, TestSize.Level1) sqlite3 *db = nullptr; EXPECT_EQ(GetDB(db), SQLITE_OK); - std::string sql = "DELETE FROM TEST_TABLE WHERE ID = 0"; + std::string sql = "DELETE FROM TEST_TABLE WHERE 1 = 1"; EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db, sql), E_OK); sqlite3_close(db); @@ -486,13 +708,13 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync004, TestSize.Level1) */ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync005, TestSize.Level1) { - RowData rowData; - PrepareVirtualEnvironment(rowData); + std::map dataMap; + PrepareVirtualEnvironment(dataMap); Query query = Query::Select(g_tableName); g_deviceB->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, true); - CheckData(rowData); + CheckData(dataMap); } /** @@ -504,12 +726,12 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync005, TestSize.Level1) */ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync006, TestSize.Level1) { - RowData rowData; - PrepareVirtualEnvironment(rowData); + std::map dataMap; + PrepareVirtualEnvironment(dataMap); BlockSync(SYNC_MODE_PULL_ONLY, OK); - CheckData(rowData); + CheckData(dataMap); } /** @@ -524,8 +746,8 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync001, TestSize.Leve /** * @tc.steps: step1. open rdb store, create distribute table and insert data */ - RowData rowData; - PrepareVirtualEnvironment(rowData); + std::map dataMap; + PrepareVirtualEnvironment(dataMap); /** * @tc.steps: step2. set auto launch callBack @@ -544,8 +766,8 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync001, TestSize.Leve /** * @tc.steps: step3. close store ensure communicator has closed */ - g_mgr.CloseStore(g_kvDelegatePtr); - g_kvDelegatePtr = nullptr; + g_mgr.CloseStore(g_rdbDelegatePtr); + g_rdbDelegatePtr = nullptr; /** * @tc.steps: step4. RunCommunicatorLackCallback to autolaunch store */ @@ -560,12 +782,8 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync001, TestSize.Leve /** * @tc.steps: step6. check sync data ensure sync successful */ - CheckData(rowData); - - RelationalStoreDelegate::Option option; - g_mgr.OpenStore(g_dbDir, STORE_ID_1, option, g_kvDelegatePtr); - ASSERT_TRUE(g_kvDelegatePtr != nullptr); - + CheckData(dataMap); + OpenStore(); std::this_thread::sleep_for(std::chrono::minutes(1)); } @@ -582,8 +800,8 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync002, TestSize.Leve /** * @tc.steps: step1. open rdb store, create distribute table and insert data */ - RowData rowData; - PrepareVirtualEnvironment(rowData); + std::map dataMap; + PrepareVirtualEnvironment(dataMap); /** * @tc.steps: step2. set auto launch callBack @@ -595,8 +813,8 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync002, TestSize.Leve /** * @tc.steps: step2. close store ensure communicator has closed */ - g_mgr.CloseStore(g_kvDelegatePtr); - g_kvDelegatePtr = nullptr; + g_mgr.CloseStore(g_rdbDelegatePtr); + g_rdbDelegatePtr = nullptr; /** * @tc.steps: step3. store cann't autoLaunch because callback return false */ @@ -630,15 +848,15 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync003, TestSize.Leve /** * @tc.steps: step1. open rdb store, create distribute table and insert data */ - RowData rowData; - PrepareVirtualEnvironment(rowData); + std::map dataMap; + PrepareVirtualEnvironment(dataMap); g_mgr.SetAutoLaunchRequestCallback(nullptr); /** * @tc.steps: step2. close store ensure communicator has closed */ - g_mgr.CloseStore(g_kvDelegatePtr); - g_kvDelegatePtr = nullptr; + g_mgr.CloseStore(g_rdbDelegatePtr); + g_rdbDelegatePtr = nullptr; /** * @tc.steps: step3. store cann't autoLaunch because callback is nullptr */ @@ -659,4 +877,133 @@ HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AutoLaunchSync003, TestSize.Leve OpenStore(); std::this_thread::sleep_for(std::chrono::minutes(1)); } + +/** +* @tc.name: Ability Sync 001 +* @tc.desc: Test ability sync success when has same schema. +* @tc.type: FUNC +* @tc.require: AR000GK58N +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AbilitySync001, TestSize.Level1) +{ + std::map dataMap; + std::vector localFieldInfo; + GetFieldInfo(localFieldInfo, g_storageType); + + PrepareEnvironment(dataMap, localFieldInfo, localFieldInfo); + BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); + + CheckVirtualData(dataMap); +} + +/** +* @tc.name: Ability Sync 002 +* @tc.desc: Test ability sync failed when has different schema. +* @tc.type: FUNC +* @tc.require: AR000GK58N +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, AbilitySync002, TestSize.Level1) +{ + /** + * @tc.steps: step1. set local schema is (BOOL, INTEGER, REAL, TEXT, BLOB, INTEGER) + */ + std::map dataMap; + std::vector localFieldInfo; + std::vector localStorageType = g_storageType; + localStorageType.push_back(StorageType::STORAGE_TYPE_INTEGER); + GetFieldInfo(localFieldInfo, localStorageType); + + /** + * @tc.steps: step2. set remote schema is (BOOL, INTEGER, REAL, TEXT, BLOB, TEXT) + */ + std::vector remoteFieldInfo; + std::vector remoteStorageType = g_storageType; + remoteStorageType.push_back(StorageType::STORAGE_TYPE_TEXT); + GetFieldInfo(remoteFieldInfo, remoteStorageType); + + /** + * @tc.steps: step3. call sync + * @tc.expected: sync fail when abilitySync + */ + PrepareEnvironment(dataMap, localFieldInfo, remoteFieldInfo); + BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, SCHEMA_MISMATCH); +} + +/** +* @tc.name: WaterMark 001 +* @tc.desc: Test sync success after erase waterMark. +* @tc.type: FUNC +* @tc.require: AR000GK58N +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, WaterMark001, TestSize.Level1) +{ + std::map dataMap; + PrepareEnvironment(dataMap); + BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); + + CheckVirtualData(dataMap); + + EXPECT_EQ(g_rdbDelegatePtr->RemoveDeviceData(g_deviceB->GetDeviceId(), g_tableName), OK); + g_deviceB->EraseSyncData(g_tableName); + + BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); + + CheckVirtualData(dataMap); +} + +/* +* @tc.name: pressure sync 001 +* @tc.desc: Test rdb sync different table at same time +* @tc.type: FUNC +* @tc.require: AR000GK58N +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, PressureSync001, TestSize.Level1) +{ + /** + * @tc.steps: step1. create table A and device A push data to device B + * @tc.expected: step1. all is ok + */ + std::map tableADataMap; + std::vector tableAFieldInfo; + std::vector localStorageType = g_storageType; + localStorageType.push_back(StorageType::STORAGE_TYPE_INTEGER); + GetFieldInfo(tableAFieldInfo, localStorageType); + const std::string tableNameA = "TABLE_A"; + PrepareEnvironment(tableADataMap, tableNameA, tableAFieldInfo, tableAFieldInfo); + + /** + * @tc.steps: step2. create table B and device B push data to device A + * @tc.expected: step2. all is ok + */ + std::map tableBDataMap; + std::vector tableBFieldInfo; + localStorageType = g_storageType; + localStorageType.push_back(StorageType::STORAGE_TYPE_REAL); + GetFieldInfo(tableBFieldInfo, localStorageType); + const std::string tableNameB = "TABLE_B"; + PrepareVirtualEnvironment(tableBDataMap, tableNameB, tableBFieldInfo); + + std::condition_variable cv; + bool subFinish = false; + std::thread subThread = std::thread([&subFinish, &cv, &tableNameA, &tableADataMap]() { + BlockSync(tableNameA, SyncMode::SYNC_MODE_PUSH_ONLY, OK); + + CheckVirtualData(tableNameA, tableADataMap); + subFinish = true; + cv.notify_all(); + }); + subThread.detach(); + + Query query = Query::Select(tableNameB); + g_deviceB->GenericVirtualDevice::Sync(SYNC_MODE_PUSH_ONLY, query, true); + CheckData(tableBDataMap, tableNameB, tableBFieldInfo); + + std::mutex mutex; + std::unique_lock lock(mutex); + cv.wait(lock, [&subFinish]{ return subFinish; }); +} #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp index d78398d32..2614f207b 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp @@ -49,5 +49,15 @@ int RelationalVirtualDevice::Sync(SyncMode mode, bool wait) { return -E_NOT_SUPPORT; } + +void RelationalVirtualDevice::EraseSyncData(const std::string &tableName) +{ + static_cast(storage_)->EraseSyncData(tableName); +} + +void RelationalVirtualDevice::SetTableInfo(const TableInfo &tableInfo) +{ + static_cast(storage_)->SetTableInfo(tableInfo); +} } // DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h index b3232f9c9..e4e1baf74 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h @@ -31,7 +31,9 @@ public: int GetAllSyncData(const std::string &tableName, std::vector &data); int GetSyncData(const std::string &tableName, const std::string &hashKey, VirtualRowData &data); void SetLocalFieldInfo(const std::vector &localFieldInfo); + void SetTableInfo(const TableInfo &tableInfo); int Sync(SyncMode mode, bool wait) override; + void EraseSyncData(const std::string &tableName); }; } #endif diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp index 1f12f5b3c..afae09798 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp @@ -71,7 +71,7 @@ int VirtualCommunicator::SendMessage(const std::string &dstTarget, const Message int VirtualCommunicator::GetRemoteCommunicatorVersion(const std::string &deviceId, uint16_t &version) const { - version = 1; + version = UINT16_MAX; return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp index c1185f423..bf1453a00 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp @@ -119,6 +119,7 @@ int VirtualRelationalVerSyncDBInterface::GetSyncData(QueryObject &query, for (const auto &[hashKey, virtualData] : localData_[query.GetTableName()]) { if (virtualData.logInfo.timestamp < timeRange.beginTime || virtualData.logInfo.timestamp >= timeRange.endTime) { + LOGD("ignore hashkey %s", hashKey.c_str()); continue; } RowDataWithLog rowData; @@ -276,6 +277,7 @@ const KvDBProperties &VirtualRelationalVerSyncDBInterface::GetDbProperties() con void VirtualRelationalVerSyncDBInterface::SetLocalFieldInfo(const std::vector &localFieldInfo) { + localFieldInfo_.clear(); // sort by dict std::map infoMap; for (const auto &item : localFieldInfo) { @@ -311,15 +313,28 @@ int VirtualRelationalVerSyncDBInterface::GetSyncData(const std::string &tableNam return E_OK; } +void VirtualRelationalVerSyncDBInterface::EraseSyncData(const std::string &tableName) +{ + if (syncData_.find(tableName) == syncData_.end()) { + return; + } + syncData_.erase(tableName); +} + int VirtualRelationalVerSyncDBInterface::CreateDistributedDeviceTable(const std::string &device, const RelationalSyncStrategy &syncStrategy) { - return -E_NOT_SUPPORT; + return E_OK; } int VirtualRelationalVerSyncDBInterface::RegisterSchemaChangedCallback(const std::function &onSchemaChanged) { - return -E_NOT_SUPPORT; + return E_OK; +} + +void VirtualRelationalVerSyncDBInterface::SetTableInfo(const TableInfo &tableInfo) +{ + schemaObj_.AddRelationalTable(tableInfo); } } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h index e4f3dbd7a..cbb9ae67a 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h @@ -97,6 +97,10 @@ public: int CreateDistributedDeviceTable(const std::string &device, const RelationalSyncStrategy &syncStrategy) override; int RegisterSchemaChangedCallback(const std::function &onSchemaChanged) override; + + void EraseSyncData(const std::string &tableName); + + void SetTableInfo(const TableInfo &tableInfo); private: -- Gitee From afd9151f88117fbfe41f4a857c42b4567e778fbb Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 10 Jan 2022 19:07:41 +0800 Subject: [PATCH 22/38] Add relational schema compare and negotiate Signed-off-by: lianhuix --- .../distributeddb/common/include/db_errno.h | 4 + .../relational/relational_schema_object.h | 10 + .../distributeddb/common/include/schema.h | 10 + .../relational/relational_schema_object.cpp | 221 +++++++++++++- .../common/src/schema_object.cpp | 10 - .../distributeddb/common/src/schema_utils.cpp | 1 + .../relational/sqlite_relational_store.cpp | 4 +- .../relational/sqlite_relational_store.h | 3 +- ...ibuteddb_relational_schema_object_test.cpp | 283 +++++++++++++++++- 9 files changed, 521 insertions(+), 25 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h index b3c6acc35..1561c2ce8 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h @@ -138,6 +138,10 @@ constexpr int E_VALUE_MISMATCH_FEILD_COUNT = (E_BASE + 182); // Value mismatch s constexpr int E_VALUE_MISMATCH_FEILD_TYPE = (E_BASE + 183); // Value mismatch schema in field type constexpr int E_VALUE_MISMATCH_CONSTRAINT = (E_BASE + 184); // Value mismatch schema in constraint constexpr int E_VALUE_MISMATCH_OTHER_REASON = (E_BASE + 185); // Value mismatch schema in other reason +constexpr int E_RELATIONAL_TABLE_EQUAL = (E_BASE + 186); // In table is same +constexpr int E_RELATIONAL_TABLE_COMPATIBLE = (E_BASE + 187); // In table is compatible +constexpr int E_RELATIONAL_TABLE_COMPATIBLE_UPGRADE = (E_BASE + 188); // In table has more fields with default value +constexpr int E_RELATIONAL_TABLE_INCOMPATIBLE = (E_BASE + 189); // In table is incompatible // Num 200+ is reserved for fixed value errno, which should not be changed between time // Message with errorNo of Feedback-type is generated by CommunicatorAggregator without data part(No deserial if exist) constexpr int E_FEEDBACK_UNKNOWN_MESSAGE = (E_BASE + 200); // Unknown message feedback from remote device diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index 6ccde2d3c..d051f5f69 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -45,6 +45,8 @@ public: // return field define string like ("fieldName": "MY INT(21), NOT NULL, DEFAULT 123") std::string ToAttributeString() const; + + int CompareWithField(const FieldInfo &inField) const; private: std::string fieldName_; std::string dataType_; // Type may be null @@ -78,10 +80,17 @@ public: void AddTrigger(const std::string &triggerName); std::string ToTableInfoString() const; void SetDevId(const std::string &devId); + + int CompareWithTable(const TableInfo &inTableInfo) const; + private: void AddFieldDefineString(std::string &attrStr) const; void AddUniqueDefineString(std::string &attrStr) const; void AddIndexDefineString(std::string &attrStr) const; + + int CompareWithTableFields(const std::map &inTableFields) const; + int CompareWithTableIndex(const std::map &inTableIndex) const; + std::string tableName_; std::string devId_; bool autoInc_ = false; // only 'INTEGER PRIMARY KEY' could be defined as 'AUTOINCREMENT' @@ -121,6 +130,7 @@ public: static RelationalSyncOpinion MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, const std::string &remoteSchemaStr, uint8_t remoteSchemaType); + // The remoteOpinion.checkOnReceive is ignored static RelationalSyncStrategy ConcludeSyncStrategy(const RelationalSyncOpinion &localOpinion, const RelationalSyncOpinion &remoteOpinion); diff --git a/services/distributeddataservice/libs/distributeddb/common/include/schema.h b/services/distributeddataservice/libs/distributeddb/common/include/schema.h index c5670b1c7..1311d08b2 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/schema.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/schema.h @@ -29,11 +29,21 @@ enum class SchemaType : uint8_t { RELATIVE = 3, UNRECOGNIZED = 4 }; + +inline SchemaType ReadSchemaType(uint8_t inType) +{ + if (inType >= static_cast(SchemaType::UNRECOGNIZED)) { + return SchemaType::UNRECOGNIZED; + } + return static_cast(inType); +} + struct SyncOpinion { bool permitSync = false; bool requirePeerConvert = false; bool checkOnReceive = false; }; + struct SyncStrategy { bool permitSync = false; bool convertOnSend = false; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index 478a3e6af..a2dda1744 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -107,6 +107,18 @@ std::string FieldInfo::ToAttributeString() const return attrStr; } +int FieldInfo::CompareWithField(const FieldInfo &inField) const +{ + if (fieldName_ != inField.GetFieldName() || dataType_ != inField.GetDataType() || + isNotNull_ != inField.IsNotNull()) { + return false; + } + if (hasDefaultValue_ && inField.HasDefaultValue()) { + return defaultValue_ == inField.GetDefaultValue(); + } + return hasDefaultValue_ == inField.HasDefaultValue(); +} + const std::string &TableInfo::GetTableName() const { return tableName_; @@ -267,6 +279,72 @@ void TableInfo::SetDevId(const std::string &devId) devId_ = devId; } +int TableInfo::CompareWithTable(const TableInfo &inTableInfo) const +{ + if (tableName_ != inTableInfo.GetTableName()) { + LOGW("[Relational][Compare] Table name is not same"); + return -E_RELATIONAL_TABLE_INCOMPATIBLE; + } + + if (primaryKey_ != inTableInfo.GetPrimaryKey()) { + LOGW("[Relational][Compare] Table primary key is not same"); + return -E_RELATIONAL_TABLE_INCOMPATIBLE; + } + + int fieldCompareResult = CompareWithTableFields(inTableInfo.GetFields()); + if (fieldCompareResult == -E_RELATIONAL_TABLE_INCOMPATIBLE) { + LOGW("[Relational][Compare] Compare table fields with in table, %d", fieldCompareResult); + return -E_RELATIONAL_TABLE_INCOMPATIBLE; + } + + int indexCompareResult = CompareWithTableIndex(inTableInfo.GetIndexDefine()); + return (fieldCompareResult == -E_RELATIONAL_TABLE_EQUAL) ? indexCompareResult : fieldCompareResult; +} + +int TableInfo::CompareWithTableFields(const std::map &inTableFields) const +{ + auto itLocal = fields_.begin(); + auto itInTable = inTableFields.begin(); + + while (itLocal != fields_.end() && itInTable != inTableFields.end()) { + if (itLocal->first == itInTable->first) { // Same field + if (!itLocal->second.CompareWithField(itInTable->second)) { // Compare field + LOGW("[Relational][Compare] Table field is incompatible"); // not compatible + return -E_RELATIONAL_TABLE_INCOMPATIBLE; + } + itLocal++; // Compare next field + } else { // Assume local table fields is a subset of in table + if (itInTable->second.IsNotNull() && !itInTable->second.HasDefaultValue()) { // Upgrade field not compatible + LOGW("[Relational][Compare] Table upgrade field should allowed to be empty or have default value."); + return -E_RELATIONAL_TABLE_INCOMPATIBLE; + } + } + itInTable++; // Next in table field + } + + if (itLocal != fields_.end()) { + LOGW("[Relational][Compare] Table field is missing"); + return -E_RELATIONAL_TABLE_INCOMPATIBLE; + } + + return (itInTable == inTableFields.end()) ? -E_RELATIONAL_TABLE_EQUAL : -E_RELATIONAL_TABLE_COMPATIBLE_UPGRADE; +} + +int TableInfo::CompareWithTableIndex(const std::map &inTableIndex) const +{ + auto itLocal = indexDefines_.begin(); + auto itInTable = inTableIndex.begin(); + while (itLocal != indexDefines_.end() && itInTable != inTableIndex.end()) { + if (itLocal != itInTable) { + return -E_RELATIONAL_TABLE_COMPATIBLE; + } + itLocal++; + itInTable++; + } + return (itLocal == indexDefines_.end() && itInTable == inTableIndex.end()) ? -E_RELATIONAL_TABLE_EQUAL : + -E_RELATIONAL_TABLE_COMPATIBLE; +} + namespace { std::string VectorJoin(const CompositeFields &fields, char con) { @@ -321,19 +399,73 @@ std::string TableInfo::ToTableInfoString() const return attrStr; } +namespace { + const std::string MAGIC = "relational_opinion"; +} + uint32_t RelationalSyncOpinion::CalculateParcelLen(uint32_t softWareVersion) const { - return E_OK; + uint64_t len = Parcel::GetStringLen(MAGIC); + len += Parcel::GetUInt32Len(); + len += Parcel::GetUInt32Len(); + len = Parcel::GetEightByteAlign(len); + for (const auto &it : opinions_) { + len += Parcel::GetStringLen(it.first); + len += Parcel::GetUInt32Len(); + len += Parcel::GetUInt32Len(); + len = Parcel::GetEightByteAlign(len); + } + if (len > UINT32_MAX) { + return 0; + } + return static_cast(len); } int RelationalSyncOpinion::SerializeData(Parcel &parcel, uint32_t softWareVersion) const { - return E_OK; + (void)parcel.WriteString(MAGIC); + (void)parcel.WriteUInt32(softWareVersion); + (void)parcel.WriteUInt32(static_cast(opinions_.size())); + (void)parcel.EightByteAlign(); + for (const auto &it : opinions_) { + (void)parcel.WriteString(it.first); + (void)parcel.WriteUInt32(it.second.permitSync); + (void)parcel.WriteUInt32(it.second.requirePeerConvert); + (void)parcel.EightByteAlign(); + } + return parcel.IsError() ? -E_INVALID_ARGS : E_OK; } int RelationalSyncOpinion::DeserializeData(Parcel &parcel, RelationalSyncOpinion &opinion) { - return E_OK; + if (!parcel.IsContinueRead()) { + return E_OK; + } + std::string magicStr; + (void)parcel.ReadString(magicStr); + if (magicStr != MAGIC) { + LOGE("Decerialize sync opinion failed while read MAGIC string [%s]", magicStr.c_str()); + return -E_INVALID_ARGS; + } + uint32_t softWareVersion; + (void)parcel.ReadUInt32(softWareVersion); + uint32_t opinionSize; + (void)parcel.ReadUInt32(opinionSize); + (void)parcel.EightByteAlign(); + for (uint32_t i = 0; i < opinionSize; i++) { + std::string tableName; + SyncOpinion tableOpinion; + (void)parcel.ReadString(tableName); + uint32_t permitSync; + (void)parcel.ReadUInt32(permitSync); + tableOpinion.permitSync = static_cast(permitSync); + uint32_t requirePeerConvert; + (void)parcel.ReadUInt32(requirePeerConvert); + tableOpinion.requirePeerConvert = static_cast(requirePeerConvert); + (void)parcel.EightByteAlign(); + opinion.AddSyncOpinion(tableName, tableOpinion); + } + return parcel.IsError() ? -E_INVALID_ARGS : E_OK; } SyncOpinion RelationalSyncOpinion::GetTableOpinion(const std::string &tableName) const @@ -372,13 +504,91 @@ void RelationalSyncStrategy::AddSyncStrategy(const std::string &tableName, const RelationalSyncOpinion RelationalSchemaObject::MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, const std::string &remoteSchemaStr, uint8_t remoteSchemaType) { - return RelationalSyncOpinion(); + SchemaType localType = localSchema.GetSchemaType(); + SchemaType remoteType = ReadSchemaType(remoteSchemaType); + + if (remoteType == SchemaType::UNRECOGNIZED) { + LOGW("[RelationalSchema][opinion] Remote schema type %d is unrecognized.", remoteSchemaType); + return {}; + } + + if (remoteType != SchemaType::RELATIVE) { + LOGW("[RelationalSchema][opinion] Not support sync with schema type: local-type=[%d] remote-type=[%d]", + SchemaUtils::SchemaTypeString(localType).c_str(), SchemaUtils::SchemaTypeString(remoteType).c_str()); + return {}; + } + + if (!localSchema.IsSchemaValid()) { + LOGW("[RelationalSchema][opinion] Local schema is not valid"); + return {}; + } + + RelationalSchemaObject remoteSchema; + int errCode = remoteSchema.ParseFromSchemaString(remoteSchemaStr); + if (errCode != E_OK) { + LOGW("[RelationalSchema][opinion] Parse remote schema failed %d, remote schema type %s", errCode, + SchemaUtils::SchemaTypeString(remoteType).c_str()); + return {}; + } + + RelationalSyncOpinion opinion; + for (const auto &it : localSchema.GetTables()) { + if (remoteSchema.GetTable(it.first).GetTableName() != it.first) { + LOGW("[RelationalSchema][opinion] Table was missing in remote schema"); + continue; + } + // remote table is compatible(equal or upgrade) based on local table, permit sync and don't need check + errCode = it.second.CompareWithTable(remoteSchema.GetTable(it.first)); + if (errCode != -E_RELATIONAL_TABLE_INCOMPATIBLE) { + opinion.AddSyncOpinion(it.first, {true, false, false}); + continue; + } + // local table is compatible upgrade based on remote table, permit sync and need check + errCode = remoteSchema.GetTable(it.first).CompareWithTable(it.second); + if (errCode != -E_RELATIONAL_TABLE_INCOMPATIBLE) { + opinion.AddSyncOpinion(it.first, {true, false, true}); + continue; + } + // local table is incompatible with remote table mutually, don't permit sync and need check + LOGW("[RelationalSchema][opinion] Local table is incompartible with remote table mutually."); + opinion.AddSyncOpinion(it.first, {false, true, true}); + } + + return opinion; } RelationalSyncStrategy RelationalSchemaObject::ConcludeSyncStrategy(const RelationalSyncOpinion &localOpinion, const RelationalSyncOpinion &remoteOpinion) { - return RelationalSyncStrategy(); + RelationalSyncStrategy syncStrategy; + for (const auto &itLocal : localOpinion.GetOpinions()) { + if (remoteOpinion.GetOpinions().find(itLocal.first) == remoteOpinion.GetOpinions().end()) { + LOGW("[RelationalSchema][Strategy] Table opinion is not found from remote."); + continue; + } + + SyncStrategy strategy; + SyncOpinion localTableOpinion = itLocal.second; + SyncOpinion remoteTableOpinion = remoteOpinion.GetTableOpinion(itLocal.first); + + strategy.permitSync = (localTableOpinion.permitSync || remoteTableOpinion.permitSync); + + bool convertConflict = (localTableOpinion.requirePeerConvert && remoteTableOpinion.requirePeerConvert); + if (convertConflict) { + // Should not both need peer convert + strategy.permitSync = false; + } + // No peer convert required means local side has convert capability, convert locally takes precedence + strategy.convertOnSend = (!localTableOpinion.requirePeerConvert); + strategy.convertOnReceive = remoteTableOpinion.requirePeerConvert; + + strategy.checkOnReceive = localTableOpinion.checkOnReceive; + LOGI("[RelationalSchema][Strategy] PermitSync=%d, SendConvert=%d, ReceiveConvert=%d, ReceiveCheck=%d.", + strategy.permitSync, strategy.convertOnSend, strategy.convertOnReceive, strategy.checkOnReceive); + syncStrategy.AddSyncStrategy(itLocal.first, strategy); + } + + return syncStrategy; } bool RelationalSchemaObject::IsSchemaValid() const @@ -757,6 +967,7 @@ int RelationalSchemaObject::ParseCheckTableIndex(const JsonObject &inJsonObject, } resultTable.AddIndexDefine(field.first[1], indexDefine); // 1 : second element in path } + tables_[resultTable.GetTableName()] = resultTable; return E_OK; } } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/schema_object.cpp index 6470c54ca..1ede64146 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/schema_object.cpp @@ -80,16 +80,6 @@ std::string SchemaObject::GenerateExtractSQL(SchemaType inSchemaType, const Fiel return resultSql; } -namespace { -inline SchemaType ReadSchemaType(uint8_t inType) -{ - if (inType >= static_cast(SchemaType::UNRECOGNIZED)) { - return SchemaType::UNRECOGNIZED; - } - return static_cast(inType); -} -} - // Some principle in current version describe below. (Relative-type will be introduced in future but not involved now) // 1. PermitSync: Be false may because schemaType-unrecognized, schemaType-different, schema-unparsable, // schemaVersion-unrecognized, schema-incompatible, and so on. diff --git a/services/distributeddataservice/libs/distributeddb/common/src/schema_utils.cpp b/services/distributeddataservice/libs/distributeddb/common/src/schema_utils.cpp index c1fe1fe96..78d40f67c 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/schema_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/schema_utils.cpp @@ -466,6 +466,7 @@ std::string SchemaUtils::SchemaTypeString(SchemaType inType) {SchemaType::NONE, "NONE"}, {SchemaType::JSON, "JSON-SCHEMA"}, {SchemaType::FLATBUFFER, "FLATBUFFER-SCHEMA"}, + {SchemaType::RELATIVE, "RELATIVE"}, {SchemaType::UNRECOGNIZED, "UNRECOGNIZED"}, }; return schemaTypeMapString[inType]; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index f012cca09..7b357c912 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -232,7 +232,7 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } - errCode = CleanDistributedDeviceTable(); + errCode = CleanDistributedDeviceTable(); // TODO: remove water mark if (errCode != E_OK) { break; } @@ -415,7 +415,7 @@ int SQLiteRelationalStore::RemoveDeviceData(const std::string &device, const std (void)handle->Rollback(); } else { (void)handle->Commit(); - errCode = syncEngine_->EraseDeviceWaterMark(device, true); + errCode = syncEngine_->EraseDeviceWaterMark(device, true); // TODO: need table name if (errCode != E_OK) { LOGE("Erase device water mark failed. %d", errCode); } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index c4f984032..aa3cafbb4 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -75,7 +75,7 @@ private: int StartLifeCycleTimer(const DatabaseLifeCycleNotifier ¬ifier) const; void HeartBeat() const; int ResetLifeCycleTimer() const; - + // use for sync Interactive std::unique_ptr syncEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv @@ -94,7 +94,6 @@ private: mutable std::mutex initalMutex_; bool isInitialized_ = false; - mutable std::mutex schemaMutex_; //lifeCycle mutable std::mutex lifeCycleMutex_; mutable DatabaseLifeCycleNotifier lifeCycleNotifier_; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp index 77f194d4e..0e9bf78a3 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp @@ -29,7 +29,7 @@ using namespace DistributedDB; using namespace DistributedDBUnitTest; namespace { - constexpr const char *NORMAL_SCHEMA = R""({ + const std::string NORMAL_SCHEMA = R""({ "SCHEMA_VERSION": "2.0", "SCHEMA_TYPE": "RELATIVE", "TABLES": [{ @@ -38,17 +38,17 @@ namespace { "field_name1": { "TYPE": "STRING", "NOT_NULL": true, - "DEFINE": "abcd" + "DEFAULT": "abcd" }, "field_name2": { "TYPE": "MYINT(21)", "NOT_NULL": false, - "DEFINE": "222" + "DEFAULT": "222" }, "field_name3": { "TYPE": "INTGER", "NOT_NULL": false, - "DEFINE": "1" + "DEFAULT": "1" } }, "AUTOINCREMENT": true, @@ -73,6 +73,98 @@ namespace { "PRIMARY_KEY": "field_name1" }] })""; + + const std::string INVALID_SCHEMA = R""({ + "SCHEMA_VERSION": "2.0", + "SCHEMA_TYPE": "RELATIVE", + "TABLES": [{ + "NAME": "FIRST", + "DEFINE": { + "field_name1": { + "TYPE": "STRING", + "NOT_NULL": true, + "DEFAULT": "abcd" + },"field_name2": { + "TYPE": "MYINT(21)", + "NOT_NULL": false, + "DEFAULT": "222" + } + }, + "PRIMARY_KEY": "field_name1" + }] + })""; + + const std::string INVALID_JSON_STRING = R""({ + "SCHEMA_VERSION": "2.0", + "SCHEMA_TYPE": "RELATIVE", + "TABLES": [{ + "NAME": "FIRST", + "DEFINE": { + "field_name1": {)""; + + const std::string SCHEMA_VERSION_STR_1 = R"("SCHEMA_VERSION": "1.0",)"; + const std::string SCHEMA_VERSION_STR_2 = R"("SCHEMA_VERSION": "2.0",)"; + const std::string SCHEMA_VERSION_STR_INVALID = R"("SCHEMA_VERSION": "awd3",)"; + const std::string SCHEMA_TYPE_STR_NONE = R"("SCHEMA_TYPE": "NONE",)"; + const std::string SCHEMA_TYPE_STR_JSON = R"("SCHEMA_TYPE": "JSON",)"; + const std::string SCHEMA_TYPE_STR_FLATBUFFER = R"("SCHEMA_TYPE": "FLATBUFFER",)"; + const std::string SCHEMA_TYPE_STR_RELATIVE = R"("SCHEMA_TYPE": "RELATIVE",)"; + const std::string SCHEMA_TYPE_STR_INVALID = R"("SCHEMA_TYPE": "adewaaSAD",)"; + + const std::string SCHEMA_TABLE_STR = R""("TABLES": [{ + "NAME": "FIRST", + "DEFINE": { + "field_name1": { + "TYPE": "STRING", + "NOT_NULL": true, + "DEFAULT": "abcd" + },"field_name2": { + "TYPE": "MYINT(21)", + "NOT_NULL": false, + "DEFAULT": "222" + } + }, + "PRIMARY_KEY": "field_name1" + }])""; + + const std::string TABLE_DEFINE_STR = R""({ + "NAME": "FIRST", + "DEFINE": { + "field_name1": { + "TYPE": "STRING", + "NOT_NULL": true, + "DEFAULT": "abcd" + },"field_name2": { + "TYPE": "MYINT(21)", + "NOT_NULL": false, + "DEFAULT": "222" + } + }, + "PRIMARY_KEY": "field_name1" + })""; + + const std::string TABLE_DEFINE_STR_NAME = R""("NAME": "FIRST",)""; + const std::string TABLE_DEFINE_STR_NAME_INVALID = R"("NAME": 123,)"; + const std::string TABLE_DEFINE_STR_FIELDS = R""("DEFINE": { + "field_name1": { + "TYPE": "STRING", + "NOT_NULL": true, + "DEFAULT": "abcd" + },"field_name2": { + "TYPE": "MYINT(21)", + "NOT_NULL": false, + "DEFAULT": "222" + } + },)""; + const std::string TABLE_DEFINE_STR_FIELDS_EMPTY = R""("DEFINE": {},)""; + const std::string TABLE_DEFINE_STR_FIELDS_NOTYPE = R""("DEFINE": { + "field_name1": { + "NOT_NULL": true, + "DEFAULT": "abcd" + }},)""; + const std::string TABLE_DEFINE_STR_KEY = R""("PRIMARY_KEY": "field_name1")""; + const std::string TABLE_DEFINE_STR_KEY_INVALID = R""("PRIMARY_KEY": false)""; + } class DistributedDBRelationalSchemaObjectTest : public testing::Test { @@ -89,13 +181,13 @@ void DistributedDBRelationalSchemaObjectTest::SetUp() } /** - * @tc.name: RelationalSchemaTest001 + * @tc.name: RelationalSchemaParseTest001 * @tc.desc: Test relational schema parse from json string * @tc.type: FUNC * @tc.require: AR000GK58I * @tc.author: lianhuix */ -HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaTest001, TestSize.Level1) +HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaParseTest001, TestSize.Level1) { const std::string schemaStr = NORMAL_SCHEMA; RelationalSchemaObject schemaObj; @@ -106,4 +198,183 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaTest001, TestS schemaObj2.ParseFromSchemaString(schemaObj.ToSchemaString()); EXPECT_EQ(errCode, E_OK); } + +/** + * @tc.name: RelationalSchemaParseTest002 + * @tc.desc: Test relational schema parse from invalid json string + * @tc.type: FUNC + * @tc.require: AR000GK58I + * @tc.author: lianhuix + */ +HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaParseTest002, TestSize.Level1) +{ + RelationalSchemaObject schemaObj; + + std::string schemaStr01(SCHEMA_STRING_SIZE_LIMIT + 1, 's'); + int errCode = schemaObj.ParseFromSchemaString(schemaStr01); + EXPECT_EQ(errCode, -E_INVALID_ARGS); + + errCode = schemaObj.ParseFromSchemaString(INVALID_JSON_STRING); + EXPECT_EQ(errCode, -E_JSON_PARSE_FAIL); + + std::string noVersion = "{" + SCHEMA_TYPE_STR_RELATIVE + SCHEMA_TABLE_STR + "}"; + errCode = schemaObj.ParseFromSchemaString(noVersion); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidVersion1 = "{" + SCHEMA_VERSION_STR_1 + SCHEMA_TYPE_STR_RELATIVE + SCHEMA_TABLE_STR + "}"; + errCode = schemaObj.ParseFromSchemaString(invalidVersion1); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidVersion2 = "{" + SCHEMA_VERSION_STR_INVALID + SCHEMA_TYPE_STR_RELATIVE + SCHEMA_TABLE_STR + "}"; + errCode = schemaObj.ParseFromSchemaString(invalidVersion2); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string noType = "{" + SCHEMA_VERSION_STR_2 + SCHEMA_TABLE_STR + "}"; + errCode = schemaObj.ParseFromSchemaString(noType); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidType1 = "{" + SCHEMA_VERSION_STR_2 + SCHEMA_TYPE_STR_NONE + SCHEMA_TABLE_STR + "}"; + errCode = schemaObj.ParseFromSchemaString(invalidType1); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidType2 = "{" + SCHEMA_VERSION_STR_2 + SCHEMA_TYPE_STR_JSON + SCHEMA_TABLE_STR + "}"; + errCode = schemaObj.ParseFromSchemaString(invalidType2); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidType3 = "{" + SCHEMA_VERSION_STR_2 + SCHEMA_TYPE_STR_FLATBUFFER + SCHEMA_TABLE_STR + "}"; + errCode = schemaObj.ParseFromSchemaString(invalidType3); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidType4 = "{" + SCHEMA_VERSION_STR_2 + SCHEMA_TYPE_STR_INVALID + SCHEMA_TABLE_STR + "}"; + errCode = schemaObj.ParseFromSchemaString(invalidType4); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string noTable = "{" + SCHEMA_VERSION_STR_2 + + SCHEMA_TYPE_STR_RELATIVE.substr(0, SCHEMA_TYPE_STR_RELATIVE.length() - 1) + "}"; + errCode = schemaObj.ParseFromSchemaString(noTable); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); +} + +namespace { +std::string GenerateFromTableStr(const std::string tableStr) +{ + return R""({ + "SCHEMA_VERSION": "2.0", + "SCHEMA_TYPE": "RELATIVE", + "TABLES": )"" + tableStr + "}"; +} +} + +/** + * @tc.name: RelationalSchemaParseTest003 + * @tc.desc: Test relational schema parse from invalid json string + * @tc.type: FUNC + * @tc.require: AR000GK58I + * @tc.author: lianhuix + */ +HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaParseTest003, TestSize.Level1) +{ + RelationalSchemaObject schemaObj; + int errCode = E_OK; + + // std::string emptyTableStr = GenerateFromTableStr("[]"); + // int errCode = schemaObj.ParseFromSchemaString(emptyTableStr); + // EXPECT_EQ(errCode, -E_INVALID_ARGS); + + errCode = schemaObj.ParseFromSchemaString(GenerateFromTableStr(TABLE_DEFINE_STR)); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + // errCode = schemaObj.ParseFromSchemaString(GenerateFromTableStr("[" + TABLE_DEFINE_STR + "]")); + // EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidTableStr01 = "{" + TABLE_DEFINE_STR_FIELDS + TABLE_DEFINE_STR_KEY + "}"; + errCode = schemaObj.ParseFromSchemaString(GenerateFromTableStr("[" + invalidTableStr01 + "]")); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidTableStr02 = "{" + TABLE_DEFINE_STR_NAME_INVALID + TABLE_DEFINE_STR_FIELDS + + TABLE_DEFINE_STR_KEY + "}"; + errCode = schemaObj.ParseFromSchemaString(GenerateFromTableStr("[" + invalidTableStr02 + "]")); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + // std::string invalidTableStr03 = "{" + TABLE_DEFINE_STR_NAME + TABLE_DEFINE_STR_FIELDS_EMPTY + + // TABLE_DEFINE_STR_KEY + "}"; + // errCode = schemaObj.ParseFromSchemaString(GenerateFromTableStr("[" + invalidTableStr03 + "]")); + // EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidTableStr04 = "{" + TABLE_DEFINE_STR_NAME + TABLE_DEFINE_STR_FIELDS_NOTYPE + + TABLE_DEFINE_STR_KEY + "}"; + errCode = schemaObj.ParseFromSchemaString(GenerateFromTableStr("[" + invalidTableStr04 + "]")); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); + + std::string invalidTableStr05 = "{" + TABLE_DEFINE_STR_NAME + TABLE_DEFINE_STR_FIELDS + + TABLE_DEFINE_STR_KEY_INVALID + "}"; + errCode = schemaObj.ParseFromSchemaString(GenerateFromTableStr("[" + invalidTableStr05 + "]")); + EXPECT_EQ(errCode, -E_SCHEMA_PARSE_FAIL); +} + +/** + * @tc.name: RelationalSchemaCompareTest001 + * @tc.desc: Test relational schema negotiate with same schema string + * @tc.type: FUNC + * @tc.require: AR000GK58I + * @tc.author: lianhuix + */ +HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaCompareTest001, TestSize.Level1) +{ + RelationalSchemaObject schemaObj; + int errCode = schemaObj.ParseFromSchemaString(NORMAL_SCHEMA); + EXPECT_EQ(errCode, E_OK); + + RelationalSyncOpinion opinion = RelationalSchemaObject::MakeLocalSyncOpinion(schemaObj, NORMAL_SCHEMA, + static_cast(SchemaType::RELATIVE)); + EXPECT_EQ(opinion.GetTableOpinion("FIRST").permitSync, true); + EXPECT_EQ(opinion.GetTableOpinion("FIRST").checkOnReceive, false); + EXPECT_EQ(opinion.GetTableOpinion("FIRST").requirePeerConvert, false); +} + +/** + * @tc.name: RelationalSchemaOpinionTest001 + * @tc.desc: Test relational schema sync opinion + * @tc.type: FUNC + * @tc.require: AR000GK58I + * @tc.author: lianhuix + */ +HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaOpinionTest001, TestSize.Level1) +{ + const uint32_t softWareVersion = 106; + RelationalSyncOpinion opinion; + opinion.AddSyncOpinion("table_1", SyncOpinion {true, false, false}); + opinion.AddSyncOpinion("table_2", SyncOpinion {false, true, false}); + opinion.AddSyncOpinion("table_3", SyncOpinion {false, false, true}); + + uint32_t len = opinion.CalculateParcelLen(softWareVersion); + std::vector buff(len, 0); + Parcel writeParcel(buff.data(), len); + int errCode = opinion.SerializeData(writeParcel, softWareVersion); + EXPECT_EQ(errCode, E_OK); + + Parcel readParcel(buff.data(), len); + RelationalSyncOpinion opinionRecv; + errCode = RelationalSyncOpinion::DeserializeData(readParcel, opinionRecv); + EXPECT_EQ(errCode, E_OK); + + EXPECT_EQ(opinion.GetOpinions().size(), opinionRecv.GetOpinions().size()); + for (const auto &it : opinion.GetOpinions()) { + SyncOpinion tableOpinionRecv = opinionRecv.GetTableOpinion(it.first); + EXPECT_EQ(it.second.permitSync, tableOpinionRecv.permitSync); + EXPECT_EQ(it.second.requirePeerConvert, tableOpinionRecv.requirePeerConvert); + } +} + +/** + * @tc.name: RelationalSchemaNegotiateTest001 + * @tc.desc: Test relational schema negotiate + * @tc.type: FUNC + * @tc.require: AR000GK58I + * @tc.author: lianhuix + */ +HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaNegotiateTest001, TestSize.Level1) +{ + +} #endif \ No newline at end of file -- Gitee From 40852f425379b0da2cd33d4af33e37c35476442c Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 13 Jan 2022 19:26:06 +0800 Subject: [PATCH 23/38] Move schema to storage engine Signed-off-by: lianhuix --- .../relational/relational_schema_object.cpp | 14 ++- .../src/relational_sync_able_storage.cpp | 2 +- .../relational/sqlite_relational_store.cpp | 97 +++------------ ...qlite_single_relational_storage_engine.cpp | 114 +++++++++++++++++- .../sqlite_single_relational_storage_engine.h | 15 +++ ...single_ver_relational_storage_executor.cpp | 11 +- 6 files changed, 161 insertions(+), 92 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index a2dda1744..4351d9223 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -252,7 +252,7 @@ void TableInfo::AddIndexDefineString(std::string &attrStr) const if (indexDefines_.empty()) { return; } - attrStr += R"("INDEX": {)"; + attrStr += R"(,"INDEX": {)"; for (auto itIndexDefine = indexDefines_.begin(); itIndexDefine != indexDefines_.end(); ++itIndexDefine) { attrStr += "\"" + (*itIndexDefine).first + "\": [\""; for (auto itField = itIndexDefine->second.begin(); itField != itIndexDefine->second.end(); ++itField) { @@ -392,7 +392,7 @@ std::string TableInfo::ToTableInfoString() const } AddUniqueDefineString(attrStr); if (!primaryKey_.empty()) { - attrStr += R"("PRIMARY_KEY": ")" + primaryKey_ + "\","; + attrStr += R"("PRIMARY_KEY": ")" + primaryKey_ + "\""; } AddIndexDefineString(attrStr); attrStr += "}"; @@ -829,7 +829,12 @@ int RelationalSchemaObject::ParseCheckTableInfo(const JsonObject &inJsonObject) if (errCode != E_OK) { return errCode; } - return ParseCheckTableIndex(inJsonObject, resultTable); + errCode = ParseCheckTableIndex(inJsonObject, resultTable); + if (errCode != E_OK) { + return errCode; + } + tables_[resultTable.GetTableName()] = resultTable; + return E_OK; } int RelationalSchemaObject::ParseCheckTableName(const JsonObject &inJsonObject, TableInfo &resultTable) @@ -867,7 +872,7 @@ int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject } FieldInfo fieldInfo; - fieldInfo.SetFieldName(field.first[0]); // 0 : first element in path + fieldInfo.SetFieldName(field.first[1]); // 1 : table name element in path errCode = ParseCheckTableFieldInfo(fieldObj, field.first, fieldInfo); if (errCode != E_OK) { LOGE("[RelationalSchema][Parse] Parse table field info failed. %d", errCode); @@ -967,7 +972,6 @@ int RelationalSchemaObject::ParseCheckTableIndex(const JsonObject &inJsonObject, } resultTable.AddIndexDefine(field.first[1], indexDefine); // 1 : second element in path } - tables_[resultTable.GetTableName()] = resultTable; return E_OK; } } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index 786272b49..3ab840e5b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -425,7 +425,7 @@ int RelationalSyncAbleStorage::RemoveDeviceData(const std::string &deviceName, b RelationalSchemaObject RelationalSyncAbleStorage::GetSchemaInfo() const { - return RelationalSchemaObject(); + return storageEngine_->GetSchemaRef(); } int RelationalSyncAbleStorage::GetSecurityOption(SecurityOption &option) const diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 7b357c912..b2a9d7fa2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -122,8 +122,7 @@ int SQLiteRelationalStore::GetSchemaFromMeta() return errCode; } - std::lock_guard lock(schemaMutex_); - properties_.SetSchema(schema); + sqliteStorageEngine_->SetSchema(schema); return E_OK; } @@ -131,7 +130,7 @@ int SQLiteRelationalStore::SaveSchemaToMeta() { const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); Value schemaVal; - DBCommon::StringToVector(properties_.GetSchema().ToSchemaString(), schemaVal); + DBCommon::StringToVector(sqliteStorageEngine_->GetSchemaRef().ToSchemaString(), schemaVal); int errCode = storageEngine_->PutMetaData(schemaKey, schemaVal); if (errCode != E_OK) { LOGE("Save relational schema to meta table failed. %d", errCode); @@ -153,38 +152,14 @@ int SQLiteRelationalStore::SaveLogTableVersionToMeta() int SQLiteRelationalStore::CleanDistributedDeviceTable() { - int errCode = E_OK; - auto handle = GetHandle(true, errCode); - if (handle == nullptr) { - return errCode; - } - - errCode = handle->StartTransaction(TransactType::IMMEDIATE); - if (errCode != E_OK) { - ReleaseHandle(handle); - return errCode; - } - - std::lock_guard lock(schemaMutex_); - RelationalSchemaObject schema = properties_.GetSchema(); std::vector missingTables; - errCode = handle->CkeckAndCleanDistributedTable(schema.GetTableNames(), missingTables); - if (errCode == E_OK) { - errCode = handle->Commit(); - if (errCode == E_OK) { - // Remove non-existent tables from the schema - for (const auto &tableName : missingTables) { - schema.RemoveRelationalTable(tableName); - } - properties_.SetSchema(schema); - } - } else { - LOGE("Check distributed table failed. %d", errCode); - (void)handle->Rollback(); + int errCode = sqliteStorageEngine_->CleanDistributedDeviceTable(missingTables); + if (errCode != E_OK) { + LOGE("Clean distributed device table failed. %d", errCode); } - - ReleaseHandle(handle); - return SaveSchemaToMeta(); + // TODO: remove water mark + // syncEngine_->EraseDeviceWaterMark(); + return errCode; } int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) @@ -232,7 +207,7 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } - errCode = CleanDistributedDeviceTable(); // TODO: remove water mark + errCode = CleanDistributedDeviceTable(); if (errCode != E_OK) { break; } @@ -342,59 +317,19 @@ void SQLiteRelationalStore::WakeUpSyncer() int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) { - int errCode = E_OK; - std::lock_guard lock(schemaMutex_); - auto schema = properties_.GetSchema(); - if (schema.GetTable(tableName).GetTableName() == tableName) { - LOGW("distributed table was already created."); - return E_OK; - } - - if (schema.GetTables().size() >= DBConstant::MAX_DISTRIBUTED_TABLE_COUNT) { - LOGW("The number of distributed tables is exceeds limit."); - return -E_MAX_LIMITS; - } - - LOGD("Create distributed table."); - auto *handle = GetHandle(true, errCode); - if (handle == nullptr) { - return errCode; - } - - errCode = handle->StartTransaction(TransactType::IMMEDIATE); + int errCode = sqliteStorageEngine_->CreateDistributedTable(tableName); if (errCode != E_OK) { - ReleaseHandle(handle); - return errCode; + LOGE("Create distributed table failed. %d", errCode); } - - TableInfo table; - errCode = handle->CreateDistributedTable(tableName, table); - if (errCode != E_OK) { - LOGE("create distributed table failed. %d", errCode); - (void)handle->Rollback(); - ReleaseHandle(handle); - return errCode; - } - errCode = handle->Commit(); - if (errCode == E_OK) { - schema.AddRelationalTable(table); - properties_.SetSchema(schema); - storageEngine_->NotifySchemaChanged(); - } - - ReleaseHandle(handle); - return SaveSchemaToMeta(); + return errCode; } int SQLiteRelationalStore::RemoveDeviceData(const std::string &device, const std::string &tableName) { - { - std::lock_guard lock(schemaMutex_); - std::map tables = properties_.GetSchema().GetTables(); - if (!tableName.empty() && tables.find(tableName) == tables.end()) { - LOGW("Remove device data with table name which is not a distributed table or not exist."); - return E_OK; - } + std::map tables = sqliteStorageEngine_->GetSchemaRef().GetTables(); + if (!tableName.empty() && tables.find(tableName) == tables.end()) { + LOGW("Remove device data with table name which is not a distributed table or not exist."); + return E_OK; } int errCode = E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index 6ef4b916e..d91d93292 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -15,10 +15,16 @@ #ifdef RELATIONAL_STORE #include "sqlite_single_relational_storage_engine.h" -#include "sqlite_single_ver_relational_storage_executor.h" +#include "db_common.h" #include "db_errno.h" +#include "sqlite_single_ver_relational_storage_executor.h" + namespace DistributedDB { +namespace { + constexpr const char *RELATIONAL_SCHEMA_KEY = "relational_schema"; +} + SQLiteSingleRelationalStorageEngine::SQLiteSingleRelationalStorageEngine() {}; SQLiteSingleRelationalStorageEngine::~SQLiteSingleRelationalStorageEngine() {}; @@ -80,5 +86,111 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage db = nullptr; return errCode; } + +int SQLiteSingleRelationalStorageEngine::ReleaseExecutor(SQLiteSingleVerRelationalStorageExecutor *&handle) +{ + if (handle == nullptr) { // TODO: check corrupted + return E_OK; + } + StorageExecutor *databaseHandle = handle; + Recycle(databaseHandle); + handle = nullptr; + return E_OK; +} + +void SQLiteSingleRelationalStorageEngine::SetSchema(const RelationalSchemaObject &schema) +{ + std::lock_guard lock(schemaMutex_); + schema_ = schema; +} + +const RelationalSchemaObject &SQLiteSingleRelationalStorageEngine::GetSchemaRef() const +{ + std::lock_guard lock(schemaMutex_); + return schema_; +} + +int SQLiteSingleRelationalStorageEngine::CreateDistributedTable(const std::string &tableName) +{ + std::lock_guard lock(schemaMutex_); + if (schema_.GetTable(tableName).GetTableName() == tableName) { + LOGW("distributed table was already created."); + // TODO: compare new schema + return E_OK; + } + + if (schema_.GetTables().size() >= DBConstant::MAX_DISTRIBUTED_TABLE_COUNT) { + LOGE("The number of distributed tables is exceeds limit."); + return -E_MAX_LIMITS; + } + + LOGD("Create distributed table."); + int errCode = E_OK; + auto *handle = static_cast(FindExecutor(true, OperatePerm::NORMAL_PERM, + errCode)); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseExecutor(handle); + return errCode; + } + + TableInfo table; + errCode = handle->CreateDistributedTable(tableName, table); + if (errCode != E_OK) { + LOGE("create distributed table failed. %d", errCode); + (void)handle->Rollback(); + ReleaseExecutor(handle); + return errCode; + } + + errCode = handle->Commit(); + if (errCode == E_OK) { + schema_.AddRelationalTable(table); + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + DBCommon::StringToVector(schema_.ToSchemaString(), schemaVal); + errCode = handle->PutKvData(schemaKey, schemaVal); // save schema to meta_data + } + ReleaseExecutor(handle); + return errCode; +} + +int SQLiteSingleRelationalStorageEngine::CleanDistributedDeviceTable(std::vector &missingTables) +{ + int errCode = E_OK; + auto handle = static_cast(FindExecutor(true, OperatePerm::NORMAL_PERM, + errCode)); + if (handle == nullptr) { + return errCode; + } + + std::lock_guard lock(schemaMutex_); + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseExecutor(handle); + return errCode; + } + + errCode = handle->CkeckAndCleanDistributedTable(schema_.GetTableNames(), missingTables); + if (errCode == E_OK) { + errCode = handle->Commit(); + if (errCode == E_OK) { + // Remove non-existent tables from the schema + for (const auto &tableName : missingTables) { + schema_.RemoveRelationalTable(tableName); + } + } + } else { + LOGE("Check distributed table failed. %d", errCode); + (void)handle->Rollback(); + } + + ReleaseExecutor(handle); + return errCode; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h index 7bb37460b..62e782757 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h @@ -29,13 +29,28 @@ public: // Delete the copy and assign constructors DISABLE_COPY_ASSIGN_MOVE(SQLiteSingleRelationalStorageEngine); + void SetSchema(const RelationalSchemaObject &schema); + + const RelationalSchemaObject &GetSchemaRef() const; + + int CreateDistributedTable(const std::string &tableName); + + int CleanDistributedDeviceTable(std::vector &missingTables); + protected: StorageExecutor *NewSQLiteStorageExecutor(sqlite3 *db, bool isWrite, bool isMemDb) override; int Upgrade(sqlite3 *db) override; int CreateNewExecutor(bool isWrite, StorageExecutor *&handle) override; private: + // For executor. + int ReleaseExecutor(SQLiteSingleVerRelationalStorageExecutor *&handle); + + // For db. int RegisterFunction(sqlite3 *db) const; + + RelationalSchemaObject schema_; + mutable std::mutex schemaMutex_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 4fb415550..4888f1a8a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -732,8 +732,8 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteDistributedDeviceTable(const std::string tablePattern = tableName.empty() ? "%" : tableName; std::string deviceTableName = DBConstant::RELATIONAL_PREFIX + tablePattern + "_" + decicePattern; - const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE " + - deviceTableName + ";"; + const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '" + + deviceTableName + "';"; sqlite3_stmt *stmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, checkSql, stmt); if (errCode != E_OK) { @@ -751,7 +751,10 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteDistributedDeviceTable(const break; } std::string realTableName; - SQLiteUtils::GetColumnTextValue(stmt, 1, realTableName); // 1: table name result column index + (void)SQLiteUtils::GetColumnTextValue(stmt, 1, realTableName); // 1: table name result column index + if (realTableName.empty()) { // sqlite might return a row with NULL + continue; + } std::string deleteSql = "DROP TABLE IF EXISTS " + realTableName + ";"; // drop the found table int errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, deleteSql); if (errCode != E_OK) { @@ -784,7 +787,7 @@ int SQLiteSingleVerRelationalStorageExecutor::CkeckAndCleanDistributedTable(cons if (tableNames.empty()) { return E_OK; } - const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name='?';"; + const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name=?;"; sqlite3_stmt *stmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, checkSql, stmt); if (errCode != E_OK) { -- Gitee From 6d883b6bb1a601b0cda023710db5645e93cae10c Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 13 Jan 2022 20:13:25 +0800 Subject: [PATCH 24/38] Add relational schema default value Signed-off-by: lianhuix --- .../common/include/relational/relational_schema_object.h | 9 +++++---- .../common/src/relational/relational_schema_object.cpp | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index d051f5f69..0c60a1d4a 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -16,10 +16,11 @@ #define RELATIONAL_SCHEMA_OBJECT_H #ifdef RELATIONAL_STORE #include -#include "parcel.h" -#include "schema.h" #include "data_value.h" #include "json_object.h" +#include "parcel.h" +#include "schema.h" +#include "schema_utils.h" namespace DistributedDB { using CompositeFields = std::vector; @@ -176,9 +177,9 @@ private: void GenerateSchemaString(); bool isValid_ = false; // set to true after parse success from string or add at least one relational table - SchemaType schemaType_ = SchemaType::NONE; // Default NONE + SchemaType schemaType_ = SchemaType::RELATIVE; // Default RELATIVE std::string schemaString_; // The minified and valid schemaString - std::string schemaVersion_; + std::string schemaVersion_ = SCHEMA_SUPPORT_VERSION_V2; std::map tables_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index 4351d9223..54afdb0f3 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -655,6 +655,7 @@ void RelationalSchemaObject::GenerateSchemaString() void RelationalSchemaObject::AddRelationalTable(const TableInfo &tb) { tables_[tb.GetTableName()] = tb; + isValid_ = true; GenerateSchemaString(); } -- Gitee From 7a920093bae4b7c4210eb7df743990b9c9572124 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Sat, 15 Jan 2022 11:52:29 +0800 Subject: [PATCH 25/38] Erease device water mark after release handle. Signed-off-by: lianhuix --- .../sqlite/relational/sqlite_relational_store.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index b2a9d7fa2..cc8da2d3d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -348,16 +348,12 @@ int SQLiteRelationalStore::RemoveDeviceData(const std::string &device, const std if (errCode != E_OK) { LOGE("delete device data failed. %d", errCode); (void)handle->Rollback(); - } else { - (void)handle->Commit(); - errCode = syncEngine_->EraseDeviceWaterMark(device, true); // TODO: need table name - if (errCode != E_OK) { - LOGE("Erase device water mark failed. %d", errCode); - } + ReleaseHandle(handle); + return errCode; } - + errCode = handle->Commit(); ReleaseHandle(handle); - return errCode; + return (errCode != E_OK) ? errCode : syncEngine_->EraseDeviceWaterMark(device, true); } int SQLiteRelationalStore::StopLifeCycleTimer() const -- Gitee From 8ebe79e29f520ab86771ccf5b38a4166bb80da45 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 18 Jan 2022 15:03:39 +0800 Subject: [PATCH 26/38] Upgrade device table while create distributed table again Signed-off-by: lianhuix --- ...qlite_single_relational_storage_engine.cpp | 40 +++- .../sqlite_single_relational_storage_engine.h | 2 + ...single_ver_relational_storage_executor.cpp | 192 +++++++++++++++++- ...e_single_ver_relational_storage_executor.h | 4 + .../storage/src/sqlite/sqlite_utils.cpp | 1 - ...stributeddb_interfaces_relational_test.cpp | 158 ++++++++++++++ 6 files changed, 393 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index d91d93292..c97ab8dd2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -115,8 +115,7 @@ int SQLiteSingleRelationalStorageEngine::CreateDistributedTable(const std::strin std::lock_guard lock(schemaMutex_); if (schema_.GetTable(tableName).GetTableName() == tableName) { LOGW("distributed table was already created."); - // TODO: compare new schema - return E_OK; + return UpgradeDistributedTable(tableName); } if (schema_.GetTables().size() >= DBConstant::MAX_DISTRIBUTED_TABLE_COUNT) { @@ -159,6 +158,43 @@ int SQLiteSingleRelationalStorageEngine::CreateDistributedTable(const std::strin return errCode; } +int SQLiteSingleRelationalStorageEngine::UpgradeDistributedTable(const std::string &tableName) +{ + LOGD("Upgrade distributed table."); + int errCode = E_OK; + auto *handle = static_cast(FindExecutor(true, OperatePerm::NORMAL_PERM, + errCode)); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseExecutor(handle); + return errCode; + } + + TableInfo newTable; + errCode = handle->UpgradeDistributedTable(schema_.GetTable(tableName), newTable); + if (errCode != E_OK) { + LOGE("Upgrade distributed table failed. %d", errCode); + (void)handle->Rollback(); + ReleaseExecutor(handle); + return errCode; + } + + errCode = handle->Commit(); + if (errCode == E_OK) { + schema_.AddRelationalTable(newTable); + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + DBCommon::StringToVector(schema_.ToSchemaString(), schemaVal); + errCode = handle->PutKvData(schemaKey, schemaVal); // save schema to meta_data + } + ReleaseExecutor(handle); + return errCode; +} + int SQLiteSingleRelationalStorageEngine::CleanDistributedDeviceTable(std::vector &missingTables) { int errCode = E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h index 62e782757..ddc238c35 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h @@ -49,6 +49,8 @@ private: // For db. int RegisterFunction(sqlite3 *db) const; + int UpgradeDistributedTable(const std::string &tableName); + RelationalSchemaObject schema_; mutable std::mutex schemaMutex_; }; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 4888f1a8a..c27534748 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -25,7 +25,6 @@ SQLiteSingleVerRelationalStorageExecutor::SQLiteSingleVerRelationalStorageExecut int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, TableInfo &table) { if (dbHandle_ == nullptr) { - LOGE("[CreateDistributedTable] Begin transaction failed, dbHandle is null."); return -E_INVALID_DB; } @@ -68,6 +67,197 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std:: return E_OK; } +int SQLiteSingleVerRelationalStorageExecutor::UpgradeDistributedTable(const TableInfo &tableInfo, + TableInfo &newTableInfo) +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + + int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableInfo.GetTableName(), newTableInfo); + if (errCode != E_OK) { + LOGE("[UpgreadDistributedTable] analysis table schema failed. %d", errCode); + return errCode; + } + + if (newTableInfo.GetCreateTableSql().find("WITHOUT ROWID") != std::string::npos) { + LOGE("[UpgreadDistributedTable] Not support create distributed table without rowid."); + return -E_NOT_SUPPORT; + } + + // new table should has same or compatible upgrade + errCode = tableInfo.CompareWithTable(newTableInfo); + if (errCode == -E_RELATIONAL_TABLE_INCOMPATIBLE) { + LOGE("[UpgreadDistributedTable] Not support with incompatible upgreade."); + return -E_SCHEMA_MISMATCH; + } + + errCode = AlterAuxTableForUpgrade(tableInfo, newTableInfo); + if (errCode != E_OK) { + LOGE("[UpgreadDistributedTable] Alter aux table for upgrade failed. %d", errCode); + } + + return errCode; +} + +namespace { +int GetDeviceTableName(sqlite3 *handle, const std::string &tableName, const std::string &device, + std::vector &deviceTables) +{ + if (device.empty() && tableName.empty()) { // device and table name should not both be empty + return -E_INVALID_ARGS; + } + std::string decicePattern = device.empty() ? "%" : DBCommon::TransferHashString(device); + std::string tablePattern = tableName.empty() ? "%" : tableName; + std::string deviceTableName = DBConstant::RELATIONAL_PREFIX + tablePattern + "_" + decicePattern; + + const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '" + + deviceTableName + "';"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(handle, checkSql, stmt); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; + } + + do { + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + break; + } else if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + LOGE("Get table name failed. %d", errCode); + break; + } + std::string realTableName; + errCode = SQLiteUtils::GetColumnTextValue(stmt, 0, realTableName); // 0: table name result column index + if (errCode != E_OK || realTableName.empty()) { // sqlite might return a row with NULL + continue; + } + if (realTableName.rfind("_log") == (realTableName.length() - 4)) { + continue; + } + deviceTables.emplace_back(realTableName); + } while (true); + + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} + +std::vector GetUpgreadeFields(const TableInfo &oldTableInfo, const TableInfo &newTableInfo) +{ + std::vector fields; + auto itOld = oldTableInfo.GetFields().begin(); + auto itNew = newTableInfo.GetFields().begin(); + for (; itNew != newTableInfo.GetFields().end(); itNew++) { + if (itOld == oldTableInfo.GetFields().end() || itOld->first != itNew->first) { + fields.emplace_back(itNew->second); + continue; + } + itOld++; + } + return fields; +} + +std::map GetChangedIndexes(const TableInfo &oldTableInfo, const TableInfo &newTableInfo) +{ + std::map indexes; + auto itOld = oldTableInfo.GetIndexDefine().begin(); + auto itNew = newTableInfo.GetIndexDefine().begin(); + auto itOldEnd = oldTableInfo.GetIndexDefine().end(); + auto itNewEnd = newTableInfo.GetIndexDefine().end(); + + while(itOld != itOldEnd && itNew != itNewEnd) { + if (itOld->first == itNew->first) { + if (itOld->second != itNew->second) { + indexes.insert({itNew->first, itNew->second}); + } + itOld ++; + itNew ++; + } else if (itOld->first < itNew->first) { + indexes.insert({itOld->first,{}}); + itOld ++; + } else if (itOld->first > itNew->first) { + indexes.insert({itNew->first, itNew->second}); + itNew ++; + } + } + + while(itOld != itOldEnd) { + indexes.insert({itOld->first,{}}); + itOld ++; + } + + while(itNew != itNewEnd) { + indexes.insert({itNew->first, itNew->second}); + itNew ++; + } + + return indexes; +} +} + +int SQLiteSingleVerRelationalStorageExecutor::AlterAuxTableForUpgrade(const TableInfo &oldTableInfo, + const TableInfo &newTableInfo) +{ + std::vector upgradFields = GetUpgreadeFields(oldTableInfo, newTableInfo); + std::map upgradeIndexces = GetChangedIndexes(oldTableInfo, newTableInfo); + std::vector deviceTables; + int errCode = GetDeviceTableName(dbHandle_, oldTableInfo.GetTableName(), {}, deviceTables); + if (errCode != E_OK) { + LOGE("Get device table name for alter table failed. %d", errCode); + return errCode; + } + + LOGD("Begin to alter table: upgrade fields[%d], indexces[%d], deviceTable[%d]", upgradFields.size(), + upgradeIndexces.size(), deviceTables.size()); + for (auto table : deviceTables) { + for (auto field : upgradFields) { + std::string alterSql = "ALTER TABLE " + table + " ADD " + field.GetFieldName() + " " + + field.GetDataType() + ";"; + errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, alterSql); + if (errCode != E_OK) { + LOGE("Alter table failed. %d", errCode); + return errCode; + } + } + } + + for (auto table : deviceTables) { + for (auto index : upgradeIndexces) { + if (index.first.empty()) { + continue; + } + std::string realIndexName = table + "_" + index.first; + std::string deleteIndexSql = "DROP INDEX IF EXISTS " + realIndexName; + errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, deleteIndexSql); + if (errCode != E_OK) { + LOGE("Drop index failed. %d", errCode); + return errCode; + } + + if (index.second.empty()) { // Drop index only + continue; + } + + auto it = index.second.begin(); + std::string indexDefine = *it++; + while(it != index.second.end()) { + indexDefine += ", " + *it++; + } + std::string createIndexSql = "CREATE INDEX IF NOT EXISTS " + realIndexName + " ON " + table + + "(" + indexDefine + ");"; + errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, createIndexSql); + if (errCode != E_OK) { + LOGE("Create index failed. %d", errCode); + return errCode; + } + } + } + + return E_OK; +} + int SQLiteSingleVerRelationalStorageExecutor::StartTransaction(TransactType type) { if (dbHandle_ == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h index 0f241591f..88ca902f6 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -35,6 +35,8 @@ public: int CreateDistributedTable(const std::string &tableName, TableInfo &table); + int UpgradeDistributedTable(const TableInfo &tableInfo, TableInfo &newTableInfo); + int StartTransaction(TransactType type); int Commit(); int Rollback(); @@ -82,6 +84,8 @@ private: int PrepareForSavingData(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const; int PrepareForSavingLog(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const; + int AlterAuxTableForUpgrade(const TableInfo &oldTableInfo, const TableInfo &newTableInfo); + TableInfo table_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index e44a55727..4344340a0 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1443,7 +1443,6 @@ int SQLiteUtils::CloneIndexes(sqlite3 *db, const std::string &oriTableName, cons if (errCode != E_OK) { return errCode; } - errCode = SQLiteUtils::ExecuteRawSQL(db, sql); if (errCode != E_OK) { LOGE("[SQLite] execute create table sql failed"); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp index efb46e2b6..a29d9edfe 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp @@ -15,6 +15,7 @@ #include +#include "db_common.h" #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" #include "log_print.h" @@ -305,4 +306,161 @@ HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest005, TestSize.L */ status = g_mgr.CloseStore(delegate); EXPECT_EQ(status, OK); +} + +namespace { +void TableModifyTest(const std::string &modifySql, DBStatus expect) +{ + /** + * @tc.steps:step1. Prepare db file + * @tc.expected: step1. Return OK. + */ + sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, NORMAL_CREATE_TABLE_SQL), SQLITE_OK); + + /** + * @tc.steps:step2. Open store + * @tc.expected: step2. return OK + */ + RelationalStoreDelegate *delegate = nullptr; + DBStatus status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate); + EXPECT_EQ(status, OK); + ASSERT_NE(delegate, nullptr); + + /** + * @tc.steps:step3. Create distirbuted table + * @tc.expected: step3. Create distributed table OK. + */ + EXPECT_EQ(delegate->CreateDistributedTable("sync_data"), OK); + + /** + * @tc.steps:step4. Upgrade table with modifySql + * @tc.expected: step4. return OK + */ + EXPECT_EQ(RelationalTestUtils::ExecSql(db, modifySql), SQLITE_OK); + + /** + * @tc.steps:step5. Create distirbuted table again + * @tc.expected: step5. Create distributed table return expect. + */ + EXPECT_EQ(delegate->CreateDistributedTable("sync_data"), expect); + + /** + * @tc.steps:step6. Close store + * @tc.expected: step6 Return OK. + */ + status = g_mgr.CloseStore(delegate); + EXPECT_EQ(status, OK); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); +} +} + +/** + * @tc.name: RelationalTableModifyTest001 + * @tc.desc: Test modify distributed table with compatible upgrade + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalTableModifyTest001, TestSize.Level1) +{ + TableModifyTest("ALTER TABLE sync_data ADD COLUMN add_field INTEGER;", OK); +} + +/** + * @tc.name: RelationalTableModifyTest002 + * @tc.desc: Test modify distributed table with incompatible upgrade + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalTableModifyTest002, TestSize.Level1) +{ + TableModifyTest("ALTER TABLE sync_data ADD COLUMN add_field INTEGER NOT NULL;", SCHEMA_MISMATCH); +} + +/** + * @tc.name: RelationalTableModifyTest003 + * @tc.desc: Test modify distributed table with incompatible upgrade + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalTableModifyTest003, TestSize.Level1) +{ + TableModifyTest("ALTER TABLE sync_data DROP COLUMN w_timestamp;", SCHEMA_MISMATCH); +} + +namespace { +void CreateDeviceTable(sqlite3 *db, const std::string &table, const std::string &device) +{ + ASSERT_NE(db, nullptr); + std::string deviceTable = DBCommon::GetDistributedTableName(device, table); + EXPECT_EQ(SQLiteUtils::CreateSameStuTable(db, table, deviceTable, false), E_OK); + EXPECT_EQ(SQLiteUtils::CloneIndexes(db, table, deviceTable), E_OK); +} +} + +/** + * @tc.name: RelationalTableModifyTest004 + * @tc.desc: Test upgrade distributed table with device table exists + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalTableModifyTest004, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare db file + * @tc.expected: step1. Return OK. + */ + sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, NORMAL_CREATE_TABLE_SQL), SQLITE_OK); + CreateDeviceTable(db, "sync_data", "DEVICE_A"); + CreateDeviceTable(db, "sync_data", "DEVICE_B"); + CreateDeviceTable(db, "sync_data", "DEVICE_C"); + + /** + * @tc.steps:step2. Open store + * @tc.expected: step2. return OK + */ + RelationalStoreDelegate *delegate = nullptr; + DBStatus status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate); + EXPECT_EQ(status, OK); + ASSERT_NE(delegate, nullptr); + + /** + * @tc.steps:step3. Create distirbuted table + * @tc.expected: step3. Create distributed table OK. + */ + EXPECT_EQ(delegate->CreateDistributedTable("sync_data"), OK); + + /** + * @tc.steps:step4. Upgrade table + * @tc.expected: step4. return OK + */ + std::string modifySql = "ALTER TABLE sync_data ADD COLUMN add_field INTEGER;"; + EXPECT_EQ(RelationalTestUtils::ExecSql(db, modifySql), SQLITE_OK); + std::string indexSql = "CREATE INDEX add_index ON sync_data (add_field);"; + EXPECT_EQ(RelationalTestUtils::ExecSql(db, indexSql), SQLITE_OK); + std::string deleteIndexSql = "DROP INDEX IF EXISTS key_index"; + EXPECT_EQ(RelationalTestUtils::ExecSql(db, deleteIndexSql), SQLITE_OK); + + /** + * @tc.steps:step5. Create distirbuted table again + * @tc.expected: step5. Create distributed table return expect. + */ + EXPECT_EQ(delegate->CreateDistributedTable("sync_data"), OK); + + /** + * @tc.steps:step6. Close store + * @tc.expected: step6 Return OK. + */ + status = g_mgr.CloseStore(delegate); + EXPECT_EQ(status, OK); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); } \ No newline at end of file -- Gitee From 24ba230acde7175e8be199213c0f6345ddb155a4 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 18 Jan 2022 17:32:26 +0800 Subject: [PATCH 27/38] Fix issue: save schema to meta while cleaned distributed table. Signed-off-by: lianhuix --- .../relational/sqlite_single_relational_storage_engine.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index c97ab8dd2..d0dbefe72 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -219,6 +219,10 @@ int SQLiteSingleRelationalStorageEngine::CleanDistributedDeviceTable(std::vector for (const auto &tableName : missingTables) { schema_.RemoveRelationalTable(tableName); } + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + DBCommon::StringToVector(schema_.ToSchemaString(), schemaVal); + errCode = handle->PutKvData(schemaKey, schemaVal); // save schema to meta_data } } else { LOGE("Check distributed table failed. %d", errCode); -- Gitee From fd3b86fe5aabe2e42c291888cb099efc8afc2905 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 18 Jan 2022 20:16:43 +0800 Subject: [PATCH 28/38] Add CreateDistributedDeviceTable impl Signed-off-by: lianhuix --- .../relational/relational_schema_object.h | 1 + .../relational/relational_schema_object.cpp | 5 ++++ .../src/relational_sync_able_storage.cpp | 23 +++++++++++++++-- ...single_ver_relational_storage_executor.cpp | 25 +++++++++++++++++++ ...e_single_ver_relational_storage_executor.h | 2 ++ 5 files changed, 54 insertions(+), 2 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index 0c60a1d4a..a96fa8a42 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -120,6 +120,7 @@ class RelationalSyncStrategy { public: SyncStrategy GetTableStrategy(const std::string &tableName) const; void AddSyncStrategy(const std::string &tableName, const SyncStrategy &strategy); + const std::map &GetStrategies() const; private: std::map strategies_; }; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index 54afdb0f3..b442894f7 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -501,6 +501,11 @@ void RelationalSyncStrategy::AddSyncStrategy(const std::string &tableName, const strategies_[tableName] = strategy; } +const std::map &RelationalSyncStrategy::GetStrategies() const +{ + return strategies_; +} + RelationalSyncOpinion RelationalSchemaObject::MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, const std::string &remoteSchemaStr, uint8_t remoteSchemaType) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index 3ab840e5b..7e16ccdfd 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -15,9 +15,10 @@ #ifdef RELATIONAL_STORE #include "relational_sync_able_storage.h" +#include "db_common.h" #include "data_compression.h" -#include "platform_specific.h" #include "generic_single_ver_kv_entry.h" +#include "platform_specific.h" #include "runtime_context.h" namespace DistributedDB { @@ -474,7 +475,25 @@ int RelationalSyncAbleStorage::SchemaChanged(int notifyEvent) int RelationalSyncAbleStorage::CreateDistributedDeviceTable(const std::string &device, const RelationalSyncStrategy &syncStrategy) { - return E_OK; + int errCode = E_OK; + auto *handle = GetHandle(true, errCode, OperatePerm::NORMAL_PERM); + if (handle == nullptr) { + return errCode; + } + for (const auto &[table, strategy] : syncStrategy.GetStrategies()) { + if (!strategy.permitSync) { + continue; + } + + errCode = handle->CreateDistributedDeviceTable(device, table); + if (errCode != E_OK) { + LOGE("Create distributed device table failed. %d", errCode); + break; + } + } + + ReleaseHandle(handle); + return errCode; } int RelationalSyncAbleStorage::RegisterSchemaChangedCallback(const std::function &callback) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index c27534748..0fbb1aa58 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -1014,5 +1014,30 @@ int SQLiteSingleVerRelationalStorageExecutor::CkeckAndCleanDistributedTable(cons SQLiteUtils::ResetStatement(stmt, true, errCode); return CheckCorruptedStatus(errCode); } + +int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedDeviceTable(const std::string &device, + const std::string &tableName) +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + + if (device.empty() || tableName.empty()) { + return -E_INVALID_ARGS; + } + + std::string deviceTableName = DBCommon::GetDistributedTableName(device, tableName); + int errCode = SQLiteUtils::CreateSameStuTable(dbHandle_, tableName, deviceTableName, false); + if (errCode != E_OK) { + LOGE("Create device table failed. %d", errCode); + return errCode; + } + + errCode = SQLiteUtils::CloneIndexes(dbHandle_, tableName, deviceTableName); + if (errCode != E_OK) { + LOGE("Copy index to device table failed. %d", errCode); + } + return errCode; +} } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h index 88ca902f6..d64ff3408 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -68,6 +68,8 @@ public: int CkeckAndCleanDistributedTable(const std::vector &tableNames, std::vector &missingTables); + int CreateDistributedDeviceTable(const std::string &device, const std::string &tableName); + private: int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData) const; -- Gitee From 4a4c88af6e5a96fd6aa8d4ae2401f78a6fd0b9df Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 18 Jan 2022 20:59:37 +0800 Subject: [PATCH 29/38] Impl get identifier from storage for sync interface Signed-off-by: lianhuix --- .../distributeddb/storage/src/relational_sync_able_storage.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index 7e16ccdfd..7ec56c272 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -58,7 +58,8 @@ void RelationalSyncAbleStorage::DecRefCount() // Get the identifier of this kvdb. std::vector RelationalSyncAbleStorage::GetIdentifier() const { - return {}; + std::string identifier = storageEngine_->GetIdentifier(); + return std::vector(identifier.begin(), identifier.end()); } // Get the max timestamp of all entries in database. -- Gitee From 492b854ff5c36ebb2eb1ad61babf8a5c01d4dfdb Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 19 Jan 2022 10:49:46 +0800 Subject: [PATCH 30/38] Add default primary key to rowid Signed-off-by: lianhuix --- ...single_ver_relational_storage_executor.cpp | 2 +- .../storage/src/sqlite/sqlite_utils.cpp | 4 ++ ...stributeddb_interfaces_relational_test.cpp | 62 +++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 0fbb1aa58..97f9ccd1a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -61,7 +61,7 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std:: // add trigger errCode = SQLiteUtils::AddRelationalLogTableTrigger(dbHandle_, table); if (errCode != E_OK) { - LOGE("[CreateDistributedTable] create log table failed"); + LOGE("[CreateDistributedTable] Add relational log table trigger failed."); return errCode; } return E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 4344340a0..205fcf9e7 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -761,6 +761,10 @@ int AnalysisSchemaFieldDefine(sqlite3 *db, const std::string &tableName, TableIn } } while (true); + if (table.GetPrimaryKey().empty()) { + table.SetPrimaryKey("rowid"); + } + SQLiteUtils::ResetStatement(statement, true, errCode); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp index a29d9edfe..10a34ba59 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp @@ -44,6 +44,18 @@ namespace { "w_timestamp INT," \ "UNIQUE(device, ori_device));" \ "CREATE INDEX key_index ON sync_data (key, flag);"; + + const std::string CREATE_TABLE_SQL_NO_PRIMARY_KEY = "CREATE TABLE IF NOT EXISTS sync_data(" \ + "key BLOB NOT NULL UNIQUE," \ + "value BLOB," \ + "timestamp INT NOT NULL," \ + "flag INT NOT NULL," \ + "device BLOB," \ + "ori_device BLOB," \ + "hash_key BLOB NOT NULL," \ + "w_timestamp INT," \ + "UNIQUE(device, ori_device));" \ + "CREATE INDEX key_index ON sync_data (key, flag);"; } class DistributedInterfacesRelationalTest : public testing::Test { @@ -308,6 +320,56 @@ HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest005, TestSize.L EXPECT_EQ(status, OK); } +/** + * @tc.name: RelationalStoreTest005 + * @tc.desc: Test create distributed table with non primary key schema + * @tc.type: FUNC + * @tc.require: AR000GK58F + * @tc.author: lianhuix + */ +HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest006, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare db file + * @tc.expected: step1. Return OK. + */ + sqlite3 *db = RelationalTestUtils::CreateDataBase(g_dbDir + STORE_ID + DB_SUFFIX); + ASSERT_NE(db, nullptr); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); + EXPECT_EQ(RelationalTestUtils::ExecSql(db, CREATE_TABLE_SQL_NO_PRIMARY_KEY), SQLITE_OK); + EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); + + /** + * @tc.steps:step2. Open store + * @tc.expected: step2. return OK + */ + RelationalStoreDelegate *delegate = nullptr; + DBStatus status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate); + EXPECT_EQ(status, OK); + ASSERT_NE(delegate, nullptr); + + /** + * @tc.steps:step3. Create distirbuted table with invald table name + * @tc.expected: step3. Create distributed table failed. + */ + EXPECT_EQ(delegate->CreateDistributedTable("sync_data"), OK); + + /** + * @tc.steps:step4. Close store + * @tc.expected: step4. Return OK. + */ + status = g_mgr.CloseStore(delegate); + EXPECT_EQ(status, OK); + delegate = nullptr; + + status = g_mgr.OpenStore(g_dbDir + STORE_ID + DB_SUFFIX, STORE_ID, {}, delegate); + EXPECT_EQ(status, OK); + ASSERT_NE(delegate, nullptr); + + status = g_mgr.CloseStore(delegate); + EXPECT_EQ(status, OK); +} + namespace { void TableModifyTest(const std::string &modifySql, DBStatus expect) { -- Gitee From 978ddb201376bc8727677ec33d2c6e83bc51ee51 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 19 Jan 2022 11:54:10 +0800 Subject: [PATCH 31/38] Add COLUMN_ID for relational schema Signed-off-by: lianhuix --- .../relational/relational_schema_object.cpp | 18 +++++++++++++----- ...ributeddb_relational_schema_object_test.cpp | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index b442894f7..6d3e0df9a 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -97,6 +97,7 @@ void FieldInfo::SetColumnId(int cid) std::string FieldInfo::ToAttributeString() const { std::string attrStr = "\"" + fieldName_ + "\": {"; + attrStr += "\"COLUMN_ID\":" + std::to_string(cid_) + ","; attrStr += "\"TYPE\":\"" + dataType_ + "\","; attrStr += "\"NOT_NULL\":" + std::string(isNotNull_ ? "true" : "false"); if (hasDefaultValue_) { @@ -890,24 +891,31 @@ int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject } int RelationalSchemaObject::ParseCheckTableFieldInfo(const JsonObject &inJsonObject, const FieldPath &path, - FieldInfo &table) + FieldInfo &field) { FieldValue fieldValue; - int errCode = GetMemberFromJsonObject(inJsonObject, "TYPE", FieldType::LEAF_FIELD_STRING, true, fieldValue); + int errCode = GetMemberFromJsonObject(inJsonObject, "COLUMN_ID", FieldType::LEAF_FIELD_INTEGER, true, fieldValue); if (errCode != E_OK) { return errCode; } - table.SetDataType(fieldValue.stringValue); + field.SetColumnId(fieldValue.integerValue); + + + errCode = GetMemberFromJsonObject(inJsonObject, "TYPE", FieldType::LEAF_FIELD_STRING, true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + field.SetDataType(fieldValue.stringValue); errCode = GetMemberFromJsonObject(inJsonObject, "NOT_NULL", FieldType::LEAF_FIELD_BOOL, true, fieldValue); if (errCode != E_OK) { return errCode; } - table.SetNotNull(fieldValue.boolValue); + field.SetNotNull(fieldValue.boolValue); errCode = GetMemberFromJsonObject(inJsonObject, "DEFAULT", FieldType::LEAF_FIELD_STRING, false, fieldValue); if (errCode == E_OK) { - table.SetDefaultValue(fieldValue.stringValue); + field.SetDefaultValue(fieldValue.stringValue); } else if (errCode != -E_NOT_FOUND) { return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp index 0e9bf78a3..27498e82a 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp @@ -36,16 +36,19 @@ namespace { "NAME": "FIRST", "DEFINE": { "field_name1": { + "COLUMN_ID":1, "TYPE": "STRING", "NOT_NULL": true, "DEFAULT": "abcd" }, "field_name2": { + "COLUMN_ID":2, "TYPE": "MYINT(21)", "NOT_NULL": false, "DEFAULT": "222" }, "field_name3": { + "COLUMN_ID":3, "TYPE": "INTGER", "NOT_NULL": false, "DEFAULT": "1" @@ -62,10 +65,12 @@ namespace { "NAME": "SECOND", "DEFINE": { "key": { + "COLUMN_ID":1, "TYPE": "BLOB", "NOT_NULL": true }, "value": { + "COLUMN_ID":2, "TYPE": "BLOB", "NOT_NULL": false } @@ -81,10 +86,12 @@ namespace { "NAME": "FIRST", "DEFINE": { "field_name1": { + "COLUMN_ID":1, "TYPE": "STRING", "NOT_NULL": true, "DEFAULT": "abcd" },"field_name2": { + "COLUMN_ID":2, "TYPE": "MYINT(21)", "NOT_NULL": false, "DEFAULT": "222" @@ -115,10 +122,12 @@ namespace { "NAME": "FIRST", "DEFINE": { "field_name1": { + "COLUMN_ID":1, "TYPE": "STRING", "NOT_NULL": true, "DEFAULT": "abcd" },"field_name2": { + "COLUMN_ID":2, "TYPE": "MYINT(21)", "NOT_NULL": false, "DEFAULT": "222" @@ -131,10 +140,12 @@ namespace { "NAME": "FIRST", "DEFINE": { "field_name1": { + "COLUMN_ID":1, "TYPE": "STRING", "NOT_NULL": true, "DEFAULT": "abcd" },"field_name2": { + "COLUMN_ID":2, "TYPE": "MYINT(21)", "NOT_NULL": false, "DEFAULT": "222" @@ -147,10 +158,12 @@ namespace { const std::string TABLE_DEFINE_STR_NAME_INVALID = R"("NAME": 123,)"; const std::string TABLE_DEFINE_STR_FIELDS = R""("DEFINE": { "field_name1": { + "COLUMN_ID":1, "TYPE": "STRING", "NOT_NULL": true, "DEFAULT": "abcd" },"field_name2": { + "COLUMN_ID":2, "TYPE": "MYINT(21)", "NOT_NULL": false, "DEFAULT": "222" @@ -159,6 +172,7 @@ namespace { const std::string TABLE_DEFINE_STR_FIELDS_EMPTY = R""("DEFINE": {},)""; const std::string TABLE_DEFINE_STR_FIELDS_NOTYPE = R""("DEFINE": { "field_name1": { + "COLUMN_ID":1, "NOT_NULL": true, "DEFAULT": "abcd" }},)""; -- Gitee From f1f2497b586c192f76fc738d294b65cf3bed3a87 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 19 Jan 2022 14:51:44 +0800 Subject: [PATCH 32/38] Fix remove device table issue when reopen store Signed-off-by: lianhuix --- ...single_ver_relational_storage_executor.cpp | 41 ++++--------------- ...stributeddb_interfaces_relational_test.cpp | 21 +++++----- 2 files changed, 20 insertions(+), 42 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 97f9ccd1a..f0e61d79c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -134,7 +134,7 @@ int GetDeviceTableName(sqlite3 *handle, const std::string &tableName, const std: if (errCode != E_OK || realTableName.empty()) { // sqlite might return a row with NULL continue; } - if (realTableName.rfind("_log") == (realTableName.length() - 4)) { + if (realTableName.rfind("_log") == (realTableName.length() - 4)) { // 4:suffix length of "_log" continue; } deviceTables.emplace_back(realTableName); @@ -915,46 +915,23 @@ int SQLiteSingleVerRelationalStorageExecutor::CheckDBModeForRelational() int SQLiteSingleVerRelationalStorageExecutor::DeleteDistributedDeviceTable(const std::string &device, const std::string &tableName) { - if (device.empty() && tableName.empty()) { // device and table name should not both be empty - return -E_INVALID_ARGS; - } - std::string decicePattern = device.empty() ? "%" : DBCommon::TransferHashString(device); - std::string tablePattern = tableName.empty() ? "%" : tableName; - std::string deviceTableName = DBConstant::RELATIONAL_PREFIX + tablePattern + "_" + decicePattern; - - const std::string checkSql = "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '" + - deviceTableName + "';"; - sqlite3_stmt *stmt = nullptr; - int errCode = SQLiteUtils::GetStatement(dbHandle_, checkSql, stmt); + std::vector deviceTables; + int errCode = GetDeviceTableName(dbHandle_, tableName, device, deviceTables); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(stmt, true, errCode); + LOGE("Get device table name for alter table failed. %d", errCode); return errCode; } - do { - errCode = SQLiteUtils::StepWithRetry(stmt, false); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - errCode = E_OK; - break; - } else if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - LOGE("Get table name failed. %d", errCode); - break; - } - std::string realTableName; - (void)SQLiteUtils::GetColumnTextValue(stmt, 1, realTableName); // 1: table name result column index - if (realTableName.empty()) { // sqlite might return a row with NULL - continue; - } - std::string deleteSql = "DROP TABLE IF EXISTS " + realTableName + ";"; // drop the found table + LOGD("Begin to delete device table: deviceTable[%d]", deviceTables.size()); + for (const auto &table : deviceTables) { + std::string deleteSql = "DROP TABLE IF EXISTS " + table + ";"; // drop the found table int errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, deleteSql); if (errCode != E_OK) { LOGE("Delete device data failed. %d", errCode); break; } - } while(true); - - SQLiteUtils::ResetStatement(stmt, true, errCode); - return CheckCorruptedStatus(errCode); + } + return errCode; } int SQLiteSingleVerRelationalStorageExecutor::DeleteDistributedLogTable(const std::string &tableName) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp index 10a34ba59..f4e196394 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_relational_test.cpp @@ -87,6 +87,16 @@ void DistributedInterfacesRelationalTest::TearDown(void) DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir); } +namespace { +void CreateDeviceTable(sqlite3 *db, const std::string &table, const std::string &device) +{ + ASSERT_NE(db, nullptr); + std::string deviceTable = DBCommon::GetDistributedTableName(device, table); + EXPECT_EQ(SQLiteUtils::CreateSameStuTable(db, table, deviceTable, false), E_OK); + EXPECT_EQ(SQLiteUtils::CloneIndexes(db, table, deviceTable), E_OK); +} +} + /** * @tc.name: RelationalStoreTest001 * @tc.desc: Test open store and create distributed db @@ -104,6 +114,7 @@ HWTEST_F(DistributedInterfacesRelationalTest, RelationalStoreTest001, TestSize.L ASSERT_NE(db, nullptr); EXPECT_EQ(RelationalTestUtils::ExecSql(db, "PRAGMA journal_mode=WAL;"), SQLITE_OK); EXPECT_EQ(RelationalTestUtils::ExecSql(db, NORMAL_CREATE_TABLE_SQL), SQLITE_OK); + CreateDeviceTable(db, "sync_data", "DEVICE_A"); EXPECT_EQ(sqlite3_close_v2(db), SQLITE_OK); /** @@ -455,16 +466,6 @@ HWTEST_F(DistributedInterfacesRelationalTest, RelationalTableModifyTest003, Test TableModifyTest("ALTER TABLE sync_data DROP COLUMN w_timestamp;", SCHEMA_MISMATCH); } -namespace { -void CreateDeviceTable(sqlite3 *db, const std::string &table, const std::string &device) -{ - ASSERT_NE(db, nullptr); - std::string deviceTable = DBCommon::GetDistributedTableName(device, table); - EXPECT_EQ(SQLiteUtils::CreateSameStuTable(db, table, deviceTable, false), E_OK); - EXPECT_EQ(SQLiteUtils::CloneIndexes(db, table, deviceTable), E_OK); -} -} - /** * @tc.name: RelationalTableModifyTest004 * @tc.desc: Test upgrade distributed table with device table exists -- Gitee From 9dcedde81f40708d30b0cffeade3f16f7ca48081 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 19 Jan 2022 16:59:59 +0800 Subject: [PATCH 33/38] Add erase water mark with device and table name Signed-off-by: lianhuix --- .../distributeddb/common/include/db_common.h | 2 ++ .../libs/distributeddb/common/src/db_common.cpp | 11 +++++++++++ .../relational/sqlite_relational_store.cpp | 17 +++++++++++++---- .../sqlite_single_relational_storage_engine.cpp | 5 +++++ .../storage/src/sqlite/sqlite_utils.cpp | 13 +++++++++++++ .../storage/src/sqlite/sqlite_utils.h | 2 ++ .../storage/src/sync_able_engine.cpp | 3 +-- 7 files changed, 47 insertions(+), 6 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_common.h b/services/distributeddataservice/libs/distributeddb/common/include/db_common.h index 8d2d793da..9b8c44003 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_common.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_common.h @@ -56,6 +56,8 @@ public: static std::string StringMasking(const std::string &oriStr, size_t remain = 3); // remain 3 unmask static std::string GetDistributedTableName(const std::string &device, const std::string &tableName); + + static void GetDeviceFromName(const std::string &deviceTableName, std::string &deviceHash, std::string &tableName); }; // Define short macro substitute for original long expression for convenience of using diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp index 81a52ea55..80847f13b 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp @@ -317,4 +317,15 @@ std::string DBCommon::GetDistributedTableName(const std::string &device, const s std::string deviceHashHex = DBCommon::TransferStringToHex(DBCommon::TransferHashString(device)); return DBConstant::RELATIONAL_PREFIX + tableName + "_" + deviceHashHex; } + +void DBCommon::GetDeviceFromName(const std::string &deviceTableName, std::string &deviceHash, std::string &tableName) +{ + std::size_t found = deviceTableName.rfind('_'); + if (found != std::string::npos && found + 1 < deviceTableName.length() && + found > DBConstant::RELATIONAL_PREFIX.length()) { + deviceHash = deviceTableName.substr(found + 1); + tableName = deviceTableName.substr(DBConstant::RELATIONAL_PREFIX.length(), + found - DBConstant::RELATIONAL_PREFIX.length()); + } +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index cc8da2d3d..dcfd72896 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -157,8 +157,16 @@ int SQLiteRelationalStore::CleanDistributedDeviceTable() if (errCode != E_OK) { LOGE("Clean distributed device table failed. %d", errCode); } - // TODO: remove water mark - // syncEngine_->EraseDeviceWaterMark(); + for (const auto &deviceTableName : missingTables) { + std::string deviceHash; + std::string tableName; + DBCommon::GetDeviceFromName(deviceTableName, deviceHash, tableName); + syncEngine_->EraseDeviceWaterMark(deviceHash, false, tableName); + if (errCode != E_OK) { + LOGE("Erase water mark failed:%d", errCode); + return errCode; + } + } return errCode; } @@ -191,6 +199,8 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } + syncEngine_ = std::make_unique(storageEngine_); + errCode = CheckDBMode(); if (errCode != E_OK) { break; @@ -212,7 +222,6 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } - syncEngine_ = std::make_unique(storageEngine_); isInitialized_ = true; return E_OK; } while (false); @@ -353,7 +362,7 @@ int SQLiteRelationalStore::RemoveDeviceData(const std::string &device, const std } errCode = handle->Commit(); ReleaseHandle(handle); - return (errCode != E_OK) ? errCode : syncEngine_->EraseDeviceWaterMark(device, true); + return (errCode != E_OK) ? errCode : syncEngine_->EraseDeviceWaterMark(device, true, tableName); } int SQLiteRelationalStore::StopLifeCycleTimer() const diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index d0dbefe72..8fc55c266 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -63,6 +63,11 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage return errCode; } do { + errCode = SQLiteUtils::SetPersistWalMode(db); + if (errCode != E_OK) { + break; + } + errCode = Upgrade(db); // cerate meta_data table. if (errCode != E_OK) { break; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 205fcf9e7..8d26cf4e1 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -2057,4 +2057,17 @@ int SQLiteUtils::CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool SQLiteUtils::ResetStatement(stmt, true, errCode); return SQLiteUtils::MapSQLiteErrno(errCode); } + +int SQLiteUtils::SetPersistWalMode(sqlite3 *db) +{ + if (db == nullptr) { + return -E_INVALID_ARGS; + } + int opCode = 1; + int errCode = sqlite3_file_control(db, "main", SQLITE_FCNTL_PERSIST_WAL, &opCode); + if (errCode != SQLITE_OK) { + LOGE("Set persist wal mode failed. %d", errCode); + } + return SQLiteUtils::MapSQLiteErrno(errCode); +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index ea6478b62..7200c2e50 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -182,6 +182,8 @@ public: static int CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool &isEmpty); + static int SetPersistWalMode(sqlite3 *db); + private: static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp, bool setWal); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp index f558c1781..da5c9b02d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -86,8 +86,7 @@ uint64_t SyncAbleEngine::GetTimeStamp() return syncer_.GetTimeStamp(); } -int SyncAbleEngine::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, - const std::string &tableName) +int SyncAbleEngine::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash, const std::string &tableName) { if (!started_) { StartSyncer(); -- Gitee From 818a41d5a5b074ef9b6fe1b1a92836d684411c30 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 19 Jan 2022 17:09:20 +0800 Subject: [PATCH 34/38] Add UT for sync strategy Signed-off-by: lianhuix --- ...istributeddb_relational_schema_object_test.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp index 27498e82a..50237b124 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp @@ -389,6 +389,19 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaOpinionTest001 */ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaNegotiateTest001, TestSize.Level1) { - + RelationalSyncOpinion localOpinion; + localOpinion.AddSyncOpinion("table_1", SyncOpinion {true, false, false}); + localOpinion.AddSyncOpinion("table_2", SyncOpinion {false, true, false}); + localOpinion.AddSyncOpinion("table_3", SyncOpinion {false, false, true}); + + RelationalSyncOpinion remoteOpinion; + remoteOpinion.AddSyncOpinion("table_2", SyncOpinion {true, false, false}); + remoteOpinion.AddSyncOpinion("table_3", SyncOpinion {false, true, false}); + remoteOpinion.AddSyncOpinion("table_4", SyncOpinion {false, false, true}); + RelationalSyncStrategy strategy = RelationalSchemaObject::ConcludeSyncStrategy(localOpinion, remoteOpinion); + + EXPECT_EQ(strategy.GetStrategies().size(), 2); + EXPECT_EQ(strategy.GetTableStrategy("table_2").permitSync, true); + EXPECT_EQ(strategy.GetTableStrategy("table_3").permitSync, false); } #endif \ No newline at end of file -- Gitee From 4407a017537595736c096eb53c55c1e8ed023bf2 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 19 Jan 2022 17:36:59 +0800 Subject: [PATCH 35/38] Fix table compare issue Signed-off-by: lianhuix --- .../relational/relational_schema_object.cpp | 2 +- ...ibuteddb_relational_schema_object_test.cpp | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index 6d3e0df9a..cea326100 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -336,7 +336,7 @@ int TableInfo::CompareWithTableIndex(const std::mapfirst != itInTable->first || itLocal->second != itInTable->second) { return -E_RELATIONAL_TABLE_COMPATIBLE; } itLocal++; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp index 50237b124..b502fd9ca 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_relational_schema_object_test.cpp @@ -346,6 +346,37 @@ HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalSchemaCompareTest001 EXPECT_EQ(opinion.GetTableOpinion("FIRST").requirePeerConvert, false); } +/** + * @tc.name: RelationalTableCompareTest001 + * @tc.desc: Test relational schema negotiate with same schema string + * @tc.type: FUNC + * @tc.require: AR000GK58I + * @tc.author: lianhuix + */ +HWTEST_F(DistributedDBRelationalSchemaObjectTest, RelationalTableCompareTest001, TestSize.Level1) +{ + RelationalSchemaObject schemaObj; + int errCode = schemaObj.ParseFromSchemaString(NORMAL_SCHEMA); + EXPECT_EQ(errCode, E_OK); + TableInfo table1 = schemaObj.GetTable("FIRST"); + TableInfo table2 = schemaObj.GetTable("FIRST"); + EXPECT_EQ(table1.CompareWithTable(table2), -E_RELATIONAL_TABLE_EQUAL); + + table2.AddIndexDefine("indexname", {"field_name2", "field_name1"}); + EXPECT_EQ(table1.CompareWithTable(table2), -E_RELATIONAL_TABLE_COMPATIBLE); + + TableInfo table3 = schemaObj.GetTable("SECOND"); + EXPECT_EQ(table1.CompareWithTable(table3), -E_RELATIONAL_TABLE_INCOMPATIBLE); + + TableInfo table4 = schemaObj.GetTable("FIRST"); + table4.AddField(table3.GetFields().at("value")); + EXPECT_EQ(table1.CompareWithTable(table4), -E_RELATIONAL_TABLE_COMPATIBLE_UPGRADE); + + TableInfo table5 = schemaObj.GetTable("FIRST"); + table5.AddField(table3.GetFields().at("key")); + EXPECT_EQ(table1.CompareWithTable(table5), -E_RELATIONAL_TABLE_COMPATIBLE_UPGRADE); +} + /** * @tc.name: RelationalSchemaOpinionTest001 * @tc.desc: Test relational schema sync opinion -- Gitee From 248fc00d7848a7023f002841585013a022a23117 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 19 Jan 2022 20:39:13 +0800 Subject: [PATCH 36/38] Use same transaction to save schema into meta Signed-off-by: lianhuix --- ...qlite_single_relational_storage_engine.cpp | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index 8fc55c266..c99470e0e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -115,15 +115,30 @@ const RelationalSchemaObject &SQLiteSingleRelationalStorageEngine::GetSchemaRef( return schema_; } +namespace { +int SaveSchemaToMetaTable(SQLiteSingleVerRelationalStorageExecutor *handle, const RelationalSchemaObject &schema) +{ + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + DBCommon::StringToVector(schema.ToSchemaString(), schemaVal); + int errCode = handle->PutKvData(schemaKey, schemaVal); // save schema to meta_data + if (errCode != E_OK) { + LOGE("Save schema to meta table failed. %d", errCode); + } + return errCode; +} +} + int SQLiteSingleRelationalStorageEngine::CreateDistributedTable(const std::string &tableName) { std::lock_guard lock(schemaMutex_); - if (schema_.GetTable(tableName).GetTableName() == tableName) { + RelationalSchemaObject tmpSchema = schema_; + if (tmpSchema.GetTable(tableName).GetTableName() == tableName) { LOGW("distributed table was already created."); return UpgradeDistributedTable(tableName); } - if (schema_.GetTables().size() >= DBConstant::MAX_DISTRIBUTED_TABLE_COUNT) { + if (tmpSchema.GetTables().size() >= DBConstant::MAX_DISTRIBUTED_TABLE_COUNT) { LOGE("The number of distributed tables is exceeds limit."); return -E_MAX_LIMITS; } @@ -151,13 +166,18 @@ int SQLiteSingleRelationalStorageEngine::CreateDistributedTable(const std::strin return errCode; } + tmpSchema.AddRelationalTable(table); + errCode = SaveSchemaToMetaTable(handle, tmpSchema); + if (errCode != E_OK) { + LOGE("Save schema to meta table for create distributed table failed. %d", errCode); + (void)handle->Rollback(); + ReleaseExecutor(handle); + return errCode; + } + errCode = handle->Commit(); if (errCode == E_OK) { - schema_.AddRelationalTable(table); - const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); - Value schemaVal; - DBCommon::StringToVector(schema_.ToSchemaString(), schemaVal); - errCode = handle->PutKvData(schemaKey, schemaVal); // save schema to meta_data + schema_ = tmpSchema; } ReleaseExecutor(handle); return errCode; @@ -166,6 +186,7 @@ int SQLiteSingleRelationalStorageEngine::CreateDistributedTable(const std::strin int SQLiteSingleRelationalStorageEngine::UpgradeDistributedTable(const std::string &tableName) { LOGD("Upgrade distributed table."); + RelationalSchemaObject tmpSchema = schema_; int errCode = E_OK; auto *handle = static_cast(FindExecutor(true, OperatePerm::NORMAL_PERM, errCode)); @@ -180,7 +201,7 @@ int SQLiteSingleRelationalStorageEngine::UpgradeDistributedTable(const std::stri } TableInfo newTable; - errCode = handle->UpgradeDistributedTable(schema_.GetTable(tableName), newTable); + errCode = handle->UpgradeDistributedTable(tmpSchema.GetTable(tableName), newTable); if (errCode != E_OK) { LOGE("Upgrade distributed table failed. %d", errCode); (void)handle->Rollback(); @@ -188,13 +209,18 @@ int SQLiteSingleRelationalStorageEngine::UpgradeDistributedTable(const std::stri return errCode; } + tmpSchema.AddRelationalTable(newTable); + errCode = SaveSchemaToMetaTable(handle, tmpSchema); + if (errCode != E_OK) { + LOGE("Save schema to meta table for upgrade distributed table failed. %d", errCode); + (void)handle->Rollback(); + ReleaseExecutor(handle); + return errCode; + } + errCode = handle->Commit(); if (errCode == E_OK) { - schema_.AddRelationalTable(newTable); - const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); - Value schemaVal; - DBCommon::StringToVector(schema_.ToSchemaString(), schemaVal); - errCode = handle->PutKvData(schemaKey, schemaVal); // save schema to meta_data + schema_ = tmpSchema; } ReleaseExecutor(handle); return errCode; -- Gitee From 80e01cce77935a25a11c8f927c5f358aff0c40db Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 19 Jan 2022 21:13:24 +0800 Subject: [PATCH 37/38] Rename syncEngine_ Signed-off-by: lianhuix --- .../sqlite/relational/sqlite_relational_store.cpp | 12 ++++++------ .../src/sqlite/relational/sqlite_relational_store.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index dcfd72896..108cdf420 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -161,7 +161,7 @@ int SQLiteRelationalStore::CleanDistributedDeviceTable() std::string deviceHash; std::string tableName; DBCommon::GetDeviceFromName(deviceTableName, deviceHash, tableName); - syncEngine_->EraseDeviceWaterMark(deviceHash, false, tableName); + syncAbleEngine_->EraseDeviceWaterMark(deviceHash, false, tableName); if (errCode != E_OK) { LOGE("Erase water mark failed:%d", errCode); return errCode; @@ -199,7 +199,7 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } - syncEngine_ = std::make_unique(storageEngine_); + syncAbleEngine_ = std::make_unique(storageEngine_); errCode = CheckDBMode(); if (errCode != E_OK) { @@ -265,7 +265,7 @@ void SQLiteRelationalStore::ReleaseHandle(SQLiteSingleVerRelationalStorageExecut int SQLiteRelationalStore::Sync(const ISyncer::SyncParma &syncParam) { - return syncEngine_->Sync(syncParam); + return syncAbleEngine_->Sync(syncParam); } // Called when a connection released. @@ -291,7 +291,7 @@ void SQLiteRelationalStore::DecreaseConnectionCounter() } // Sync Close - syncEngine_->Close(); + syncAbleEngine_->Close(); if (sqliteStorageEngine_ != nullptr) { delete sqliteStorageEngine_; @@ -320,7 +320,7 @@ void SQLiteRelationalStore::ReleaseDBConnection(RelationalStoreConnection *conne void SQLiteRelationalStore::WakeUpSyncer() { - syncEngine_->WakeUpSyncer(); + syncAbleEngine_->WakeUpSyncer(); } @@ -362,7 +362,7 @@ int SQLiteRelationalStore::RemoveDeviceData(const std::string &device, const std } errCode = handle->Commit(); ReleaseHandle(handle); - return (errCode != E_OK) ? errCode : syncEngine_->EraseDeviceWaterMark(device, true, tableName); + return (errCode != E_OK) ? errCode : syncAbleEngine_->EraseDeviceWaterMark(device, true, tableName); } int SQLiteRelationalStore::StopLifeCycleTimer() const diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index aa3cafbb4..8faa46611 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -77,7 +77,7 @@ private: int ResetLifeCycleTimer() const; // use for sync Interactive - std::unique_ptr syncEngine_ = nullptr; // For storage operate sync function + std::unique_ptr syncAbleEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv RelationalSyncAbleStorage *storageEngine_ = nullptr; // For storage operate data SQLiteSingleRelationalStorageEngine *sqliteStorageEngine_ = nullptr; -- Gitee From c8f1b3feaa7b3811ca7a869d85deb065e77ae0ee Mon Sep 17 00:00:00 2001 From: lidwchn Date: Wed, 19 Jan 2022 21:34:51 +0800 Subject: [PATCH 38/38] Fix query issue. Signed-off-by: lidwchn --- .../sqlite_single_ver_relational_storage_executor.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index f0e61d79c..43905a88c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -632,17 +632,13 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryOb return errCode; } - std::string colName; std::string dataFormat; - const std::map fields = table_.GetFields(); - for (const auto &field : fields) { - colName += field.first + ","; + for (size_t i = 0; i < table_.GetFields().size(); ++i) { dataFormat += "?,"; } - colName.pop_back(); dataFormat.pop_back(); - std::string sql = "INSERT OR REPLACE INTO " + tableName + " (" + colName + ") VALUES (" + dataFormat + ");"; + std::string sql = "INSERT OR REPLACE INTO " + tableName + " VALUES (" + dataFormat + ");"; errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); if (errCode != E_OK) { LOGE("[info statement] Get statement fail!"); -- Gitee