From 2c8eae2dc9fbb1399f8e8bc329e618c909205003 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Fri, 8 Apr 2022 11:49:38 +0800 Subject: [PATCH 01/22] Fix reviewbot. Signed-off-by: lidwchn --- .../storage/src/data_transformer.h | 163 +++++++++--------- .../sqlite_single_ver_natural_store.cpp | 1 + ...single_ver_relational_storage_executor.cpp | 12 +- .../distributeddb_communicator_common.h | 2 +- 4 files changed, 88 insertions(+), 90 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.h b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.h index 05022ff75..7a48276f7 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.h @@ -1,83 +1,82 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef DATA_TRANSFORMER_H -#define DATA_TRANSFORMER_H -#ifdef RELATIONAL_STORE - -#include -#include -#include "data_value.h" -#include "db_types.h" -#include "relational_schema_object.h" - -namespace DistributedDB { -using RowData = std::vector; -using OptRowData = std::vector; - -struct LogInfo { - int64_t dataKey = -1; - std::string device; - std::string originDev; - Timestamp timestamp = 0; - Timestamp wTimestamp = 0; - uint64_t flag = 0; - Key hashKey; // primary key hash value -}; - -struct RowDataWithLog { - LogInfo logInfo; - RowData rowData; -}; - -struct OptRowDataWithLog { - LogInfo logInfo; - OptRowData optionalData; -}; - -struct TableDataWithLog { - std::string tableName; - std::vector dataList; -}; - -struct OptTableDataWithLog { - std::string tableName; - std::vector dataList; -}; - -class DataTransformer { -public: - static int TransformTableData(const TableDataWithLog &tableDataWithLog, - const std::vector &fieldInfoList, std::vector &dataItems); - static int TransformDataItem(const std::vector &dataItems, const std::vector &remoteFieldInfo, - const std::vector &localFieldInfo, OptTableDataWithLog &tableDataWithLog); - - static int SerializeDataItem(const RowDataWithLog &data, const std::vector &fieldInfo, - DataItem &dataItem); - static int DeSerializeDataItem(const DataItem &dataItem, OptRowDataWithLog &data, - const std::vector &remoteFieldInfo); - static void ReduceMapping(const std::vector &remoteFieldInfo, - const std::vector &localFieldInfo); - -private: - static int SerializeValue(Value &value, const RowData &rowData, const std::vector &fieldInfoList); - static int DeSerializeValue(const Value &value, OptRowData &optionalData, - const std::vector &remoteFieldInfo); - - static uint32_t CalDataValueLength(const DataValue &dataValue); -}; -} - -#endif +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATA_TRANSFORMER_H +#define DATA_TRANSFORMER_H +#ifdef RELATIONAL_STORE + +#include +#include "data_value.h" +#include "db_types.h" +#include "relational_schema_object.h" + +namespace DistributedDB { +using RowData = std::vector; +using OptRowData = std::vector; + +struct LogInfo { + int64_t dataKey = -1; + std::string device; + std::string originDev; + Timestamp timestamp = 0; + Timestamp wTimestamp = 0; + uint64_t flag = 0; + Key hashKey; // primary key hash value +}; + +struct RowDataWithLog { + LogInfo logInfo; + RowData rowData; +}; + +struct OptRowDataWithLog { + LogInfo logInfo; + OptRowData optionalData; +}; + +struct TableDataWithLog { + std::string tableName; + std::vector dataList; +}; + +struct OptTableDataWithLog { + std::string tableName; + std::vector dataList; +}; + +class DataTransformer { +public: + static int TransformTableData(const TableDataWithLog &tableDataWithLog, + const std::vector &fieldInfoList, std::vector &dataItems); + static int TransformDataItem(const std::vector &dataItems, const std::vector &remoteFieldInfo, + const std::vector &localFieldInfo, OptTableDataWithLog &tableDataWithLog); + + static int SerializeDataItem(const RowDataWithLog &data, const std::vector &fieldInfo, + DataItem &dataItem); + static int DeSerializeDataItem(const DataItem &dataItem, OptRowDataWithLog &data, + const std::vector &remoteFieldInfo); + static void ReduceMapping(const std::vector &remoteFieldInfo, + const std::vector &localFieldInfo); + +private: + static int SerializeValue(Value &value, const RowData &rowData, const std::vector &fieldInfoList); + static int DeSerializeValue(const Value &value, OptRowData &optionalData, + const std::vector &remoteFieldInfo); + + static uint32_t CalDataValueLength(const DataValue &dataValue); +}; +} + +#endif #endif // DATA_TRANSFORMER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp index f43bb8677..173a55fa5 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp @@ -1605,6 +1605,7 @@ RegisterFuncType SQLiteSingleVerNaturalStore::GetFuncType(int index, const Trans } if (transMap[mid].index > index) { end = mid - 1; + continue; } return transMap[mid].funcType; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index c1d487850..42bc14fe7 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -1071,15 +1071,13 @@ int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataByQuery(std::vector Date: Tue, 12 Apr 2022 14:45:35 +0800 Subject: [PATCH 02/22] Fix wrong size in log. Signed-off-by: lidwchn --- .../src/sqlite/sqlite_single_ver_storage_executor.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index 5e9d96c4c..cc14c2ed0 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -554,13 +554,12 @@ int AppendDataItem(std::vector &dataItems, const DataItem &item, size_ const DataSizeSpecInfo &dataSizeInfo) { // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. - dataTotalSize += SQLiteSingleVerStorageExecutor::GetDataItemSerialSize(item, appendLength); - if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || - dataItems.size() >= dataSizeInfo.packetSize) { + size_t appendSize = dataTotalSize + SQLiteSingleVerStorageExecutor::GetDataItemSerialSize(item, appendLength); + if ((appendSize > dataSizeInfo.blockSize && !dataItems.empty()) || dataItems.size() >= dataSizeInfo.packetSize) { return -E_UNFINISHED; - } else { - dataItems.push_back(item); } + dataItems.push_back(item); + dataTotalSize = appendSize; return E_OK; } -- Gitee From eb5b5d4a7e6e9565dd6377e74993bc2bfef5c6a7 Mon Sep 17 00:00:00 2001 From: zqq Date: Tue, 12 Apr 2022 15:54:36 +0800 Subject: [PATCH 03/22] fix ut bug Signed-off-by: zqq --- ...stributeddb_single_ver_multi_user_test.cpp | 160 +++++++----------- 1 file changed, 61 insertions(+), 99 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_user_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_user_test.cpp index 24f3b2163..aa3614c4b 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_user_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_multi_user_test.cpp @@ -228,6 +228,65 @@ bool AutoLaunchCallBack(const std::string &identifier, AutoLaunchParam ¶m, K param.option.syncDualTupleMode = true; return ret; } + +void TestSyncWithUserChange(bool wait) +{ + /** + * @tc.steps: step1. set SyncActivationCheckCallback and only userId1 can active + */ + g_mgr1.SetSyncActivationCheckCallback(g_syncActivationCheckCallback2); + /** + * @tc.steps: step2. openstore1 in dual tuple sync mode and openstore2 in normal sync mode + * @tc.expected: step2. only user2 sync mode is active + */ + OpenStore1(true); + OpenStore2(true); + /** + * @tc.steps: step3. set SyncActivationCheckCallback and only userId2 can active + */ + g_mgr1.SetSyncActivationCheckCallback(g_syncActivationCheckCallback1); + + /** + * @tc.steps: step4. call NotifyUserChanged and block sync db concurrently + * @tc.expected: step4. return OK + */ + CipherPassword passwd; + bool startSync = false; + std::condition_variable cv; + thread subThread([&]() { + std::mutex notifyLock; + std::unique_lock lck(notifyLock); + cv.wait(lck, [&startSync]() { return startSync; }); + EXPECT_TRUE(KvStoreDelegateManager::NotifyUserChanged() == OK); + }); + subThread.detach(); + g_communicatorAggregator->RegOnDispatch([&](const std::string&, Message *inMsg) { + if (!startSync) { + startSync = true; + cv.notify_all(); + } + }); + + /** + * @tc.steps: step5. deviceA call sync and wait + * @tc.expected: step5. sync should return OK. + */ + std::map result; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + DBStatus status = g_tool.SyncTest(g_kvDelegatePtr1, devices, SYNC_MODE_PUSH_ONLY, result, wait); + EXPECT_EQ(status, OK); + g_communicatorAggregator->RegOnDispatch(nullptr); + /** + * @tc.expected: step6. onComplete should be called, and status is USER_CHANGED + */ + EXPECT_EQ(result.size(), devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_EQ(pair.second, USER_CHANGED); + } + CloseStore(); +} } /** @@ -631,50 +690,7 @@ HWTEST_F(DistributedDBSingleVerMultiUserTest, MultiUser007, TestSize.Level0) */ HWTEST_F(DistributedDBSingleVerMultiUserTest, MultiUser008, TestSize.Level0) { - /** - * @tc.steps: step1. set SyncActivationCheckCallback and only userId1 can active - */ - g_mgr1.SetSyncActivationCheckCallback(g_syncActivationCheckCallback2); - /** - * @tc.steps: step2. openstore1 in dual tuple sync mode and openstore2 in normal sync mode - * @tc.expected: step2. only user2 sync mode is active - */ - OpenStore1(true); - OpenStore2(true); - /** - * @tc.steps: step3. set SyncActivationCheckCallback and only userId2 can active - */ - g_mgr1.SetSyncActivationCheckCallback(g_syncActivationCheckCallback1); - - /** - * @tc.steps: step4. call NotifyUserChanged and block sync db concurrently - * @tc.expected: step4. return OK - */ - CipherPassword passwd; - thread subThread([&]() { - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - EXPECT_TRUE(KvStoreDelegateManager::NotifyUserChanged() == OK); - }); - subThread.detach(); - /** - * @tc.steps: step5. deviceA call sync and wait - * @tc.expected: step5. sync should return OK. - */ - std::map result; - std::vector devices; - devices.push_back(g_deviceB->GetDeviceId()); - DBStatus status = g_tool.SyncTest(g_kvDelegatePtr1, devices, SYNC_MODE_PUSH_ONLY, result, true); - EXPECT_TRUE(status == OK); - - /** - * @tc.expected: step6. onComplete should be called, and status is USER_CHANGED - */ - EXPECT_TRUE(result.size() == devices.size()); - for (const auto &pair : result) { - LOGD("dev %s, status %d", pair.first.c_str(), pair.second); - EXPECT_TRUE(pair.second == USER_CHANGED); - } - CloseStore(); + TestSyncWithUserChange(true); } /** @@ -686,61 +702,7 @@ HWTEST_F(DistributedDBSingleVerMultiUserTest, MultiUser008, TestSize.Level0) */ HWTEST_F(DistributedDBSingleVerMultiUserTest, MultiUser009, TestSize.Level0) { - /** - * @tc.steps: step1. set SyncActivationCheckCallback and only userId1 can active - */ - g_mgr1.SetSyncActivationCheckCallback(g_syncActivationCheckCallback2); - /** - * @tc.steps: step2. openstore1 in dual tuple sync mode and openstore2 in normal sync mode - * @tc.expected: step2. only user2 sync mode is active - */ - OpenStore1(true); - OpenStore2(true); - /** - * @tc.steps: step3. set SyncActivationCheckCallback and only userId2 can active - */ - g_mgr1.SetSyncActivationCheckCallback(g_syncActivationCheckCallback1); - - /** - * @tc.steps: step4. call NotifyUserChanged and block sync db concurrently - * @tc.expected: step4. return OK - */ - CipherPassword passwd; - bool startSync = false; - std::condition_variable cv; - thread subThread([&]() { - std::mutex notifyLock; - std::unique_lock lck(notifyLock); - cv.wait(lck, [&startSync]() { return startSync; }); - EXPECT_TRUE(KvStoreDelegateManager::NotifyUserChanged() == OK); - }); - subThread.detach(); - g_communicatorAggregator->RegOnDispatch([&](const std::string&, Message *inMsg) { - if (!startSync) { - startSync = true; - cv.notify_all(); - } - }); - - /** - * @tc.steps: step5. deviceA call sync and wait - * @tc.expected: step5. sync should return OK. - */ - std::map result; - std::vector devices; - devices.push_back(g_deviceB->GetDeviceId()); - DBStatus status = g_tool.SyncTest(g_kvDelegatePtr1, devices, SYNC_MODE_PUSH_ONLY, result, true); - EXPECT_EQ(status, OK); - g_communicatorAggregator->RegOnDispatch(nullptr); - /** - * @tc.expected: step6. onComplete should be called, and status is USER_CHANGED - */ - EXPECT_EQ(result.size(), devices.size()); - for (const auto &pair : result) { - LOGD("dev %s, status %d", pair.first.c_str(), pair.second); - EXPECT_EQ(pair.second, USER_CHANGED); - } - CloseStore(); + TestSyncWithUserChange(false); } /** -- Gitee From 471821257f0b9e51aa22b453eecadd9d1ccf0788 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 12 Apr 2022 16:25:35 +0800 Subject: [PATCH 04/22] Change line ending with LF Signed-off-by: lianhuix --- .../syncer/src/single_ver_relational_syncer.h | 118 +-- .../distributeddb/syncer/src/sync_config.h | 90 +-- .../syncer/relational_virtual_device.cpp | 124 +-- .../common/syncer/relational_virtual_device.h | 80 +- ...rtual_relational_ver_sync_db_interface.cpp | 712 +++++++++--------- ...virtual_relational_ver_sync_db_interface.h | 250 +++--- 6 files changed, 687 insertions(+), 687 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h index cd54e6d4f..9360e0f3a 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h @@ -1,60 +1,60 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef RELATIONAL_SYNCER_H -#define RELATIONAL_SYNCER_H -#ifdef RELATIONAL_STORE -#include "single_ver_syncer.h" -namespace DistributedDB { -class SingleVerRelationalSyncer final : public SingleVerSyncer { -public: - SingleVerRelationalSyncer() = default; - ~SingleVerRelationalSyncer() override = default; - - int Initialize(ISyncInterface *syncInterface, bool isNeedActive) override; - - // Sync function. use SyncParma to reduce parameter. - int Sync(const SyncParma ¶m) override; - - void EnableAutoSync(bool enable) override; - - void LocalDataChanged(int notifyEvent) override; - -protected: - - int PrepareSync(const SyncParma ¶m, uint32_t syncId) override; - - int SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, - const std::vector &devices) const override; - -private: - - int GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, - const std::vector &tablesQuery, std::set &subSyncIdSet); - - void DoRollBack(std::set &subSyncIdSet); - - void DoOnComplete(const SyncParma ¶m, uint32_t syncId); - void DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, - const SyncParma ¶m, const std::map &devicesMap); - - void SchemaChangeCallback(); - - mutable std::mutex syncMapLock_; - std::map> fullSyncIdMap_; - std::map>> resMap_; -}; -} -#endif +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_SYNCER_H +#define RELATIONAL_SYNCER_H +#ifdef RELATIONAL_STORE +#include "single_ver_syncer.h" +namespace DistributedDB { +class SingleVerRelationalSyncer final : public SingleVerSyncer { +public: + SingleVerRelationalSyncer() = default; + ~SingleVerRelationalSyncer() override = default; + + int Initialize(ISyncInterface *syncInterface, bool isNeedActive) override; + + // Sync function. use SyncParma to reduce parameter. + int Sync(const SyncParma ¶m) override; + + void EnableAutoSync(bool enable) override; + + void LocalDataChanged(int notifyEvent) override; + +protected: + + int PrepareSync(const SyncParma ¶m, uint32_t syncId) override; + + int SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, + const std::vector &devices) const override; + +private: + + int GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, + const std::vector &tablesQuery, std::set &subSyncIdSet); + + void DoRollBack(std::set &subSyncIdSet); + + void DoOnComplete(const SyncParma ¶m, uint32_t syncId); + void DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, + const SyncParma ¶m, const std::map &devicesMap); + + void SchemaChangeCallback(); + + mutable std::mutex syncMapLock_; + std::map> fullSyncIdMap_; + std::map>> resMap_; +}; +} +#endif #endif // RELATIONAL_SYNCER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_config.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_config.h index 67b39b16a..dc2c252de 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_config.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_config.h @@ -1,46 +1,46 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef SYNC_CONFIG_H -#define SYNC_CONFIG_H - -#include -#include -#include -#include "macro_utils.h" -#include "parcel.h" -#include "types_export.h" - -// db ability config -namespace DistributedDB { -// offset, used_bits_num, used_bits_num < 64 -using AbilityItem = std::pair; -// format: {offset, used_bits_num} -/* -if need to add new ability, just add append to the last ability -current ability format: -|first bit|second bit|third bit| -|DATABASE_COMPRESSION_ZLIB|ALLPREDICATEQUERY|SUBSCRIBEQUERY| -*/ -class SyncConfig final { -public: - static const AbilityItem DATABASE_COMPRESSION_ZLIB; - static const AbilityItem ALLPREDICATEQUERY; - static const AbilityItem SUBSCRIBEQUERY; - static const AbilityItem INKEYS_QUERY; - static const std::vector ABILITYBITS; - static const std::map COMPRESSALGOMAP; -}; -} +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SYNC_CONFIG_H +#define SYNC_CONFIG_H + +#include +#include +#include +#include "macro_utils.h" +#include "parcel.h" +#include "types_export.h" + +// db ability config +namespace DistributedDB { +// offset, used_bits_num, used_bits_num < 64 +using AbilityItem = std::pair; +// format: {offset, used_bits_num} +/* +if need to add new ability, just add append to the last ability +current ability format: +|first bit|second bit|third bit| +|DATABASE_COMPRESSION_ZLIB|ALLPREDICATEQUERY|SUBSCRIBEQUERY| +*/ +class SyncConfig final { +public: + static const AbilityItem DATABASE_COMPRESSION_ZLIB; + static const AbilityItem ALLPREDICATEQUERY; + static const AbilityItem SUBSCRIBEQUERY; + static const AbilityItem INKEYS_QUERY; + static const std::vector ABILITYBITS; + static const std::map COMPRESSALGOMAP; +}; +} #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp index c4a55e158..816795b6c 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp @@ -1,63 +1,63 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifdef RELATIONAL_STORE -#include "relational_virtual_device.h" -#include "virtual_relational_ver_sync_db_interface.h" -namespace DistributedDB { -RelationalVirtualDevice::RelationalVirtualDevice(const std::string &deviceId) : GenericVirtualDevice(deviceId) -{ -} - -RelationalVirtualDevice::~RelationalVirtualDevice() -{ -} - -int RelationalVirtualDevice::PutData(const std::string &tableName, const std::vector &dataList) -{ - return static_cast(storage_)->PutLocalData(dataList, tableName); -} - -int RelationalVirtualDevice::GetAllSyncData(const std::string &tableName, std::vector &data) -{ - return static_cast(storage_)->GetAllSyncData(tableName, data); -} - -int RelationalVirtualDevice::GetSyncData(const std::string &tableName, - const std::string &hashKey, VirtualRowData &data) -{ - return static_cast(storage_)->GetVirtualSyncData(tableName, hashKey, data); -} - -void RelationalVirtualDevice::SetLocalFieldInfo(const std::vector &localFieldInfo) -{ - static_cast(storage_)->SetLocalFieldInfo(localFieldInfo); -} - -int RelationalVirtualDevice::Sync(SyncMode mode, bool wait) -{ - return -E_NOT_SUPPORT; -} - -void RelationalVirtualDevice::EraseSyncData(const std::string &tableName) -{ - static_cast(storage_)->EraseSyncData(tableName); -} - -void RelationalVirtualDevice::SetTableInfo(const TableInfo &tableInfo) -{ - static_cast(storage_)->SetTableInfo(tableInfo); -} -} // DistributedDB +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "relational_virtual_device.h" +#include "virtual_relational_ver_sync_db_interface.h" +namespace DistributedDB { +RelationalVirtualDevice::RelationalVirtualDevice(const std::string &deviceId) : GenericVirtualDevice(deviceId) +{ +} + +RelationalVirtualDevice::~RelationalVirtualDevice() +{ +} + +int RelationalVirtualDevice::PutData(const std::string &tableName, const std::vector &dataList) +{ + return static_cast(storage_)->PutLocalData(dataList, tableName); +} + +int RelationalVirtualDevice::GetAllSyncData(const std::string &tableName, std::vector &data) +{ + return static_cast(storage_)->GetAllSyncData(tableName, data); +} + +int RelationalVirtualDevice::GetSyncData(const std::string &tableName, + const std::string &hashKey, VirtualRowData &data) +{ + return static_cast(storage_)->GetVirtualSyncData(tableName, hashKey, data); +} + +void RelationalVirtualDevice::SetLocalFieldInfo(const std::vector &localFieldInfo) +{ + static_cast(storage_)->SetLocalFieldInfo(localFieldInfo); +} + +int RelationalVirtualDevice::Sync(SyncMode mode, bool wait) +{ + return -E_NOT_SUPPORT; +} + +void RelationalVirtualDevice::EraseSyncData(const std::string &tableName) +{ + static_cast(storage_)->EraseSyncData(tableName); +} + +void RelationalVirtualDevice::SetTableInfo(const TableInfo &tableInfo) +{ + static_cast(storage_)->SetTableInfo(tableInfo); +} +} // DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h index e4e1baf74..9aed0bbf3 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h @@ -1,40 +1,40 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef RELATIONAL_VIRTUAL_DEVICE_H -#define RELATIONAL_VIRTUAL_DEVICE_H -#ifdef RELATIONAL_STORE - -#include "data_transformer.h" -#include "generic_virtual_device.h" -#include "relational_schema_object.h" -#include "virtual_relational_ver_sync_db_interface.h" - -namespace DistributedDB { -class RelationalVirtualDevice final : public GenericVirtualDevice { -public: - explicit RelationalVirtualDevice(const std::string &deviceId); - ~RelationalVirtualDevice() override; - - int PutData(const std::string &tableName, const std::vector &dataList); - int GetAllSyncData(const std::string &tableName, std::vector &data); - int GetSyncData(const std::string &tableName, const std::string &hashKey, VirtualRowData &data); - void SetLocalFieldInfo(const std::vector &localFieldInfo); - void SetTableInfo(const TableInfo &tableInfo); - int Sync(SyncMode mode, bool wait) override; - void EraseSyncData(const std::string &tableName); -}; -} -#endif -#endif // RELATIONAL_VIRTUAL_DEVICE_H +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_VIRTUAL_DEVICE_H +#define RELATIONAL_VIRTUAL_DEVICE_H +#ifdef RELATIONAL_STORE + +#include "data_transformer.h" +#include "generic_virtual_device.h" +#include "relational_schema_object.h" +#include "virtual_relational_ver_sync_db_interface.h" + +namespace DistributedDB { +class RelationalVirtualDevice final : public GenericVirtualDevice { +public: + explicit RelationalVirtualDevice(const std::string &deviceId); + ~RelationalVirtualDevice() override; + + int PutData(const std::string &tableName, const std::vector &dataList); + int GetAllSyncData(const std::string &tableName, std::vector &data); + int GetSyncData(const std::string &tableName, const std::string &hashKey, VirtualRowData &data); + void SetLocalFieldInfo(const std::vector &localFieldInfo); + void SetTableInfo(const TableInfo &tableInfo); + int Sync(SyncMode mode, bool wait) override; + void EraseSyncData(const std::string &tableName); +}; +} +#endif +#endif // RELATIONAL_VIRTUAL_DEVICE_H diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp index 899d3680b..c91bf9dc1 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp @@ -1,357 +1,357 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifdef RELATIONAL_STORE -#include "db_common.h" -#include "virtual_relational_ver_sync_db_interface.h" -#include "generic_single_ver_kv_entry.h" -#include "virtual_single_ver_sync_db_Interface.h" - -namespace DistributedDB { -namespace { - int GetEntriesFromItems(std::vector &entries, const std::vector &dataItems) - { - int errCode = E_OK; - for (auto &item : dataItems) { - auto entry = new (std::nothrow) GenericSingleVerKvEntry(); - if (entry == nullptr) { - LOGE("Create entry failed."); - errCode = -E_OUT_OF_MEMORY; - break; - } - DataItem storageItem; - storageItem.key = item.key; - storageItem.value = item.value; - storageItem.flag = item.flag; - storageItem.timestamp = item.timestamp; - storageItem.writeTimestamp = item.writeTimestamp; - storageItem.hashKey = item.hashKey; - entry->SetEntryData(std::move(storageItem)); - entries.push_back(entry); - } - if (errCode != E_OK) { - LOGD("[GetEntriesFromItems] failed:%d", errCode); - for (auto &kvEntry : entries) { - delete kvEntry; - kvEntry = nullptr; - } - entries.clear(); - } - LOGD("[GetEntriesFromItems] size:%zu", dataItems.size()); - return errCode; - } - - std::string GetStr(const std::vector &vec) - { - std::string str; - DBCommon::VectorToString(vec, str); - return str; - } -} - -int VirtualRelationalVerSyncDBInterface::PutSyncDataWithQuery(const QueryObject &object, - const std::vector &entries, const std::string &deviceName) -{ - LOGD("[PutSyncData] size %zu", entries.size()); - std::vector dataItems; - for (auto itemEntry : entries) { - auto *entry = static_cast(itemEntry); - if (entry != nullptr) { - DataItem item; - item.origDev = entry->GetOrigDevice(); - item.flag = entry->GetFlag(); - item.timestamp = entry->GetTimestamp(); - item.writeTimestamp = entry->GetWriteTimestamp(); - entry->GetKey(item.key); - entry->GetValue(item.value); - entry->GetHashKey(item.hashKey); - dataItems.push_back(item); - } - } - OptTableDataWithLog optTableDataWithLog; - optTableDataWithLog.tableName = object.GetTableName(); - int errCode = DataTransformer::TransformDataItem(dataItems, localFieldInfo_, - localFieldInfo_, optTableDataWithLog); - if (errCode != E_OK) { - return errCode; - } - for (const auto &optRowDataWithLog : optTableDataWithLog.dataList) { - VirtualRowData virtualRowData; - virtualRowData.logInfo = optRowDataWithLog.logInfo; - size_t index = 0; - for (const auto &optItem : optRowDataWithLog.optionalData) { - if (index >= localFieldInfo_.size()) { - break; - } - DataValue dataValue = std::move(optItem); - LOGD("type:%d", static_cast(optItem.GetType())); - virtualRowData.objectData.PutDataValue(localFieldInfo_[index].GetFieldName(), dataValue); - index++; - } - syncData_[object.GetTableName()][GetStr(virtualRowData.logInfo.hashKey)] = virtualRowData; - } - LOGD("tableName %s", optTableDataWithLog.tableName.c_str()); - return errCode; -} - -int VirtualRelationalVerSyncDBInterface::PutLocalData(const std::vector &dataList, - const std::string &tableName) -{ - for (const auto &item : dataList) { - localData_[tableName][GetStr(item.logInfo.hashKey)] = item; - } - return E_OK; -} - -int VirtualRelationalVerSyncDBInterface::GetSyncData(QueryObject &query, - const SyncTimeRange &timeRange, const DataSizeSpecInfo &dataSizeInfo, - ContinueToken &continueStmtToken, std::vector &entries) const -{ - if (localData_.find(query.GetTableName()) == localData_.end()) { - LOGD("[GetSyncData] No Data Return"); - return E_OK; - } - std::vector dataItemList; - TableDataWithLog tableDataWithLog = {query.GetTableName(), {}}; - for (const auto &[hashKey, virtualData] : localData_[query.GetTableName()]) { - if (virtualData.logInfo.timestamp < timeRange.beginTime || - virtualData.logInfo.timestamp >= timeRange.endTime) { - LOGD("ignore hashkey %s", hashKey.c_str()); - continue; - } - RowDataWithLog rowData; - for (const auto &field : localFieldInfo_) { - DataValue dataValue; - (void)virtualData.objectData.GetDataValue(field.GetFieldName(), dataValue); - rowData.rowData.push_back(std::move(dataValue)); - } - rowData.logInfo = virtualData.logInfo; - tableDataWithLog.dataList.push_back(rowData); - } - - int errCode = DataTransformer::TransformTableData(tableDataWithLog, localFieldInfo_, dataItemList); - if (errCode != E_OK) { - return errCode; - } - continueStmtToken = nullptr; - return GetEntriesFromItems(entries, dataItemList); -} - -RelationalSchemaObject VirtualRelationalVerSyncDBInterface::GetSchemaInfo() const -{ - return schemaObj_; -} - -int VirtualRelationalVerSyncDBInterface::GetDatabaseCreateTimestamp(Timestamp &outTime) const -{ - return E_OK; -} - -int VirtualRelationalVerSyncDBInterface::GetBatchMetaData(const std::vector &keys, - std::vector &entries) const -{ - int errCode = E_OK; - for (const auto &key : keys) { - Entry entry; - entry.key = key; - errCode = GetMetaData(key, entry.value); - if (errCode != E_OK) { - return errCode; - } - entries.push_back(entry); - } - return errCode; -} - -int VirtualRelationalVerSyncDBInterface::PutBatchMetaData(std::vector &entries) -{ - int errCode = E_OK; - for (const auto &entry : entries) { - errCode = PutMetaData(entry.key, entry.value); - if (errCode != E_OK) { - return errCode; - } - } - return errCode; -} - -std::vector VirtualRelationalVerSyncDBInterface::GetTablesQuery() -{ - return {}; -} - -int VirtualRelationalVerSyncDBInterface::LocalDataChanged(int notifyEvent, std::vector &queryObj) -{ - return E_OK; -} - -int VirtualRelationalVerSyncDBInterface::GetInterfaceType() const -{ - return SYNC_RELATION; -} - -void VirtualRelationalVerSyncDBInterface::IncRefCount() -{ -} - -void VirtualRelationalVerSyncDBInterface::DecRefCount() -{ -} - -std::vector VirtualRelationalVerSyncDBInterface::GetIdentifier() const -{ - return {}; -} - -void VirtualRelationalVerSyncDBInterface::GetMaxTimestamp(Timestamp &stamp) const -{ - for (const auto &item : syncData_) { - for (const auto &entry : item.second) { - if (stamp < entry.second.logInfo.timestamp) { - stamp = entry.second.logInfo.timestamp; - } - } - } - LOGD("VirtualSingleVerSyncDBInterface::GetMaxTimestamp time = %" PRIu64, stamp); -} - -int VirtualRelationalVerSyncDBInterface::GetMetaData(const Key &key, Value &value) const -{ - auto iter = metadata_.find(key); - if (iter != metadata_.end()) { - value = iter->second; - return E_OK; - } - return -E_NOT_FOUND; -} - -int VirtualRelationalVerSyncDBInterface::PutMetaData(const Key &key, const Value &value) -{ - metadata_[key] = value; - return E_OK; -} - -int VirtualRelationalVerSyncDBInterface::DeleteMetaData(const std::vector &keys) -{ - for (const auto &key : keys) { - (void)metadata_.erase(key); - } - return E_OK; -} - -int VirtualRelationalVerSyncDBInterface::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const -{ - size_t prefixKeySize = keyPrefix.size(); - for (auto iter = metadata_.begin();iter != metadata_.end();) { - if (prefixKeySize <= iter->first.size() && - keyPrefix == Key(iter->first.begin(), std::next(iter->first.begin(), prefixKeySize))) { - iter = metadata_.erase(iter); - } else { - ++iter; - } - } - return E_OK; -} - -int VirtualRelationalVerSyncDBInterface::GetAllMetaKeys(std::vector &keys) const -{ - for (auto &iter : metadata_) { - keys.push_back(iter.first); - } - LOGD("GetAllMetaKeys size %zu", keys.size()); - return E_OK; -} - -const KvDBProperties &VirtualRelationalVerSyncDBInterface::GetDbProperties() const -{ - return properties_; -} - -void VirtualRelationalVerSyncDBInterface::SetLocalFieldInfo(const std::vector &localFieldInfo) -{ - localFieldInfo_.clear(); - localFieldInfo_ = localFieldInfo; -} - -int VirtualRelationalVerSyncDBInterface::GetAllSyncData(const std::string &tableName, - std::vector &data) -{ - if (syncData_.find(tableName) == syncData_.end()) { - return -E_NOT_FOUND; - } - for (const auto &entry : syncData_[tableName]) { - data.push_back(entry.second); - } - return E_OK; -} - -int VirtualRelationalVerSyncDBInterface::GetVirtualSyncData(const std::string &tableName, - const std::string &hashKey, VirtualRowData &data) -{ - if (syncData_.find(tableName) == syncData_.end()) { - return -E_NOT_FOUND; - } - if (syncData_.find(hashKey) == syncData_.end()) { - return -E_NOT_FOUND; - } - data = syncData_[tableName][hashKey]; - return E_OK; -} - -void VirtualRelationalVerSyncDBInterface::EraseSyncData(const std::string &tableName) -{ - if (syncData_.find(tableName) == syncData_.end()) { - return; - } - syncData_.erase(tableName); -} - -int VirtualRelationalVerSyncDBInterface::CreateDistributedDeviceTable(const std::string &device, - const RelationalSyncStrategy &syncStrategy) -{ - return E_OK; -} - -int VirtualRelationalVerSyncDBInterface::RegisterSchemaChangedCallback(const std::function &onSchemaChanged) -{ - return E_OK; -} - -void VirtualRelationalVerSyncDBInterface::SetTableInfo(const TableInfo &tableInfo) -{ - schemaObj_.AddRelationalTable(tableInfo); -} - -int VirtualRelationalVerSyncDBInterface::GetMaxTimestamp(const std::string &tableName, Timestamp ×tamp) const -{ - (void)tableName; - timestamp = 0; - return E_OK; -} - -void ObjectData::PutDataValue(const std::string &fieldName, const DataValue &value) -{ - fieldData[fieldName] = value; -} - -int ObjectData::GetDataValue(const std::string &fieldName, DataValue &value) const -{ - if (fieldData.find(fieldName) == fieldData.end()) { - return -E_NOT_FOUND; - } - value = fieldData[fieldName]; - return E_OK; -} -} +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "db_common.h" +#include "virtual_relational_ver_sync_db_interface.h" +#include "generic_single_ver_kv_entry.h" +#include "virtual_single_ver_sync_db_Interface.h" + +namespace DistributedDB { +namespace { + int GetEntriesFromItems(std::vector &entries, const std::vector &dataItems) + { + int errCode = E_OK; + for (auto &item : dataItems) { + auto entry = new (std::nothrow) GenericSingleVerKvEntry(); + if (entry == nullptr) { + LOGE("Create entry failed."); + errCode = -E_OUT_OF_MEMORY; + break; + } + DataItem storageItem; + storageItem.key = item.key; + storageItem.value = item.value; + storageItem.flag = item.flag; + storageItem.timestamp = item.timestamp; + storageItem.writeTimestamp = item.writeTimestamp; + storageItem.hashKey = item.hashKey; + entry->SetEntryData(std::move(storageItem)); + entries.push_back(entry); + } + if (errCode != E_OK) { + LOGD("[GetEntriesFromItems] failed:%d", errCode); + for (auto &kvEntry : entries) { + delete kvEntry; + kvEntry = nullptr; + } + entries.clear(); + } + LOGD("[GetEntriesFromItems] size:%zu", dataItems.size()); + return errCode; + } + + std::string GetStr(const std::vector &vec) + { + std::string str; + DBCommon::VectorToString(vec, str); + return str; + } +} + +int VirtualRelationalVerSyncDBInterface::PutSyncDataWithQuery(const QueryObject &object, + const std::vector &entries, const std::string &deviceName) +{ + LOGD("[PutSyncData] size %zu", entries.size()); + std::vector dataItems; + for (auto itemEntry : entries) { + auto *entry = static_cast(itemEntry); + if (entry != nullptr) { + DataItem item; + item.origDev = entry->GetOrigDevice(); + item.flag = entry->GetFlag(); + item.timestamp = entry->GetTimestamp(); + item.writeTimestamp = entry->GetWriteTimestamp(); + entry->GetKey(item.key); + entry->GetValue(item.value); + entry->GetHashKey(item.hashKey); + dataItems.push_back(item); + } + } + OptTableDataWithLog optTableDataWithLog; + optTableDataWithLog.tableName = object.GetTableName(); + int errCode = DataTransformer::TransformDataItem(dataItems, localFieldInfo_, + localFieldInfo_, optTableDataWithLog); + if (errCode != E_OK) { + return errCode; + } + for (const auto &optRowDataWithLog : optTableDataWithLog.dataList) { + VirtualRowData virtualRowData; + virtualRowData.logInfo = optRowDataWithLog.logInfo; + size_t index = 0; + for (const auto &optItem : optRowDataWithLog.optionalData) { + if (index >= localFieldInfo_.size()) { + break; + } + DataValue dataValue = std::move(optItem); + LOGD("type:%d", static_cast(optItem.GetType())); + virtualRowData.objectData.PutDataValue(localFieldInfo_[index].GetFieldName(), dataValue); + index++; + } + syncData_[object.GetTableName()][GetStr(virtualRowData.logInfo.hashKey)] = virtualRowData; + } + LOGD("tableName %s", optTableDataWithLog.tableName.c_str()); + return errCode; +} + +int VirtualRelationalVerSyncDBInterface::PutLocalData(const std::vector &dataList, + const std::string &tableName) +{ + for (const auto &item : dataList) { + localData_[tableName][GetStr(item.logInfo.hashKey)] = item; + } + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetSyncData(QueryObject &query, + const SyncTimeRange &timeRange, const DataSizeSpecInfo &dataSizeInfo, + ContinueToken &continueStmtToken, std::vector &entries) const +{ + if (localData_.find(query.GetTableName()) == localData_.end()) { + LOGD("[GetSyncData] No Data Return"); + return E_OK; + } + std::vector dataItemList; + TableDataWithLog tableDataWithLog = {query.GetTableName(), {}}; + for (const auto &[hashKey, virtualData] : localData_[query.GetTableName()]) { + if (virtualData.logInfo.timestamp < timeRange.beginTime || + virtualData.logInfo.timestamp >= timeRange.endTime) { + LOGD("ignore hashkey %s", hashKey.c_str()); + continue; + } + RowDataWithLog rowData; + for (const auto &field : localFieldInfo_) { + DataValue dataValue; + (void)virtualData.objectData.GetDataValue(field.GetFieldName(), dataValue); + rowData.rowData.push_back(std::move(dataValue)); + } + rowData.logInfo = virtualData.logInfo; + tableDataWithLog.dataList.push_back(rowData); + } + + int errCode = DataTransformer::TransformTableData(tableDataWithLog, localFieldInfo_, dataItemList); + if (errCode != E_OK) { + return errCode; + } + continueStmtToken = nullptr; + return GetEntriesFromItems(entries, dataItemList); +} + +RelationalSchemaObject VirtualRelationalVerSyncDBInterface::GetSchemaInfo() const +{ + return schemaObj_; +} + +int VirtualRelationalVerSyncDBInterface::GetDatabaseCreateTimestamp(Timestamp &outTime) const +{ + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetBatchMetaData(const std::vector &keys, + std::vector &entries) const +{ + int errCode = E_OK; + for (const auto &key : keys) { + Entry entry; + entry.key = key; + errCode = GetMetaData(key, entry.value); + if (errCode != E_OK) { + return errCode; + } + entries.push_back(entry); + } + return errCode; +} + +int VirtualRelationalVerSyncDBInterface::PutBatchMetaData(std::vector &entries) +{ + int errCode = E_OK; + for (const auto &entry : entries) { + errCode = PutMetaData(entry.key, entry.value); + if (errCode != E_OK) { + return errCode; + } + } + return errCode; +} + +std::vector VirtualRelationalVerSyncDBInterface::GetTablesQuery() +{ + return {}; +} + +int VirtualRelationalVerSyncDBInterface::LocalDataChanged(int notifyEvent, std::vector &queryObj) +{ + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetInterfaceType() const +{ + return SYNC_RELATION; +} + +void VirtualRelationalVerSyncDBInterface::IncRefCount() +{ +} + +void VirtualRelationalVerSyncDBInterface::DecRefCount() +{ +} + +std::vector VirtualRelationalVerSyncDBInterface::GetIdentifier() const +{ + return {}; +} + +void VirtualRelationalVerSyncDBInterface::GetMaxTimestamp(Timestamp &stamp) const +{ + for (const auto &item : syncData_) { + for (const auto &entry : item.second) { + if (stamp < entry.second.logInfo.timestamp) { + stamp = entry.second.logInfo.timestamp; + } + } + } + LOGD("VirtualSingleVerSyncDBInterface::GetMaxTimestamp time = %" PRIu64, stamp); +} + +int VirtualRelationalVerSyncDBInterface::GetMetaData(const Key &key, Value &value) const +{ + auto iter = metadata_.find(key); + if (iter != metadata_.end()) { + value = iter->second; + return E_OK; + } + return -E_NOT_FOUND; +} + +int VirtualRelationalVerSyncDBInterface::PutMetaData(const Key &key, const Value &value) +{ + metadata_[key] = value; + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::DeleteMetaData(const std::vector &keys) +{ + for (const auto &key : keys) { + (void)metadata_.erase(key); + } + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + size_t prefixKeySize = keyPrefix.size(); + for (auto iter = metadata_.begin();iter != metadata_.end();) { + if (prefixKeySize <= iter->first.size() && + keyPrefix == Key(iter->first.begin(), std::next(iter->first.begin(), prefixKeySize))) { + iter = metadata_.erase(iter); + } else { + ++iter; + } + } + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetAllMetaKeys(std::vector &keys) const +{ + for (auto &iter : metadata_) { + keys.push_back(iter.first); + } + LOGD("GetAllMetaKeys size %zu", keys.size()); + return E_OK; +} + +const KvDBProperties &VirtualRelationalVerSyncDBInterface::GetDbProperties() const +{ + return properties_; +} + +void VirtualRelationalVerSyncDBInterface::SetLocalFieldInfo(const std::vector &localFieldInfo) +{ + localFieldInfo_.clear(); + localFieldInfo_ = localFieldInfo; +} + +int VirtualRelationalVerSyncDBInterface::GetAllSyncData(const std::string &tableName, + std::vector &data) +{ + if (syncData_.find(tableName) == syncData_.end()) { + return -E_NOT_FOUND; + } + for (const auto &entry : syncData_[tableName]) { + data.push_back(entry.second); + } + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetVirtualSyncData(const std::string &tableName, + const std::string &hashKey, VirtualRowData &data) +{ + if (syncData_.find(tableName) == syncData_.end()) { + return -E_NOT_FOUND; + } + if (syncData_.find(hashKey) == syncData_.end()) { + return -E_NOT_FOUND; + } + data = syncData_[tableName][hashKey]; + return E_OK; +} + +void VirtualRelationalVerSyncDBInterface::EraseSyncData(const std::string &tableName) +{ + if (syncData_.find(tableName) == syncData_.end()) { + return; + } + syncData_.erase(tableName); +} + +int VirtualRelationalVerSyncDBInterface::CreateDistributedDeviceTable(const std::string &device, + const RelationalSyncStrategy &syncStrategy) +{ + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::RegisterSchemaChangedCallback(const std::function &onSchemaChanged) +{ + return E_OK; +} + +void VirtualRelationalVerSyncDBInterface::SetTableInfo(const TableInfo &tableInfo) +{ + schemaObj_.AddRelationalTable(tableInfo); +} + +int VirtualRelationalVerSyncDBInterface::GetMaxTimestamp(const std::string &tableName, Timestamp ×tamp) const +{ + (void)tableName; + timestamp = 0; + return E_OK; +} + +void ObjectData::PutDataValue(const std::string &fieldName, const DataValue &value) +{ + fieldData[fieldName] = value; +} + +int ObjectData::GetDataValue(const std::string &fieldName, DataValue &value) const +{ + if (fieldData.find(fieldName) == fieldData.end()) { + return -E_NOT_FOUND; + } + value = fieldData[fieldName]; + return E_OK; +} +} #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h index d82d55bbb..c2f30c829 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h @@ -1,126 +1,126 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef VIRTUAL_RELATIONAL_VER_SYNC_DB_INTERFACE_H -#define VIRTUAL_RELATIONAL_VER_SYNC_DB_INTERFACE_H -#ifdef RELATIONAL_STORE - -#include "data_transformer.h" -#include "relational_db_sync_interface.h" -#include "sqlite_single_ver_continue_token.h" -#include "relational_schema_object.h" - -namespace DistributedDB { -struct ObjectData { -public: - void PutDataValue(const std::string &fieldName, const DataValue &value); - int GetDataValue(const std::string &fieldName, DataValue &value) const; -private: - mutable std::map fieldData; -}; - -struct VirtualRowData { - LogInfo logInfo; - ObjectData objectData; -}; - -class VirtualRelationalVerSyncDBInterface : public RelationalDBSyncInterface { -public: - VirtualRelationalVerSyncDBInterface() = default; - ~VirtualRelationalVerSyncDBInterface() override = default; - - int PutSyncDataWithQuery(const QueryObject &query, const std::vector &entries, - const std::string &deviceName) override; - - int PutLocalData(const std::vector &dataList, const std::string &tableName); - - RelationalSchemaObject GetSchemaInfo() const override; - - int GetDatabaseCreateTimestamp(Timestamp &outTime) const override; - - int GetBatchMetaData(const std::vector &keys, std::vector &entries) const override; - - int PutBatchMetaData(std::vector &entries) override; - - std::vector GetTablesQuery() override; - - int LocalDataChanged(int notifyEvent, std::vector &queryObj) override; - - int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, - const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, - std::vector &entries) const override; - - int GetInterfaceType() const override; - - void IncRefCount() override; - - void DecRefCount() override; - - std::vector GetIdentifier() const override; - - void GetMaxTimestamp(Timestamp &stamp) const override; - - // Get the max timestamp of one table. - int GetMaxTimestamp(const std::string &tableName, Timestamp ×tamp) const override; - - int GetMetaData(const Key &key, Value &value) const override; - - int PutMetaData(const Key &key, const Value &value) override; - - int DeleteMetaData(const std::vector &keys) override; - - int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; - - int GetAllMetaKeys(std::vector &keys) const override; - - const KvDBProperties &GetDbProperties() const override; - - void SetLocalFieldInfo(const std::vector &localFieldInfo); - - int GetAllSyncData(const std::string &tableName, std::vector &data); - - int GetVirtualSyncData(const std::string &tableName, const std::string &hashKey, VirtualRowData &data); - - int InterceptData(std::vector &entries, - const std::string &sourceID, const std::string &targetID) const override - { - return E_OK; - } - - int CheckAndInitQueryCondition(QueryObject &query) const override - { - return E_OK; - } - - int CreateDistributedDeviceTable(const std::string &device, const RelationalSyncStrategy &syncStrategy) override; - - int RegisterSchemaChangedCallback(const std::function &onSchemaChanged) override; - - void EraseSyncData(const std::string &tableName); - - void SetTableInfo(const TableInfo &tableInfo); - -private: - mutable std::map, std::vector> metadata_; - std::map> syncData_; - mutable std::map> localData_; - std::string schema_; - RelationalSchemaObject schemaObj_; - std::vector localFieldInfo_; - KvDBProperties properties_; - SecurityOption secOption_; -}; -} -#endif +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef VIRTUAL_RELATIONAL_VER_SYNC_DB_INTERFACE_H +#define VIRTUAL_RELATIONAL_VER_SYNC_DB_INTERFACE_H +#ifdef RELATIONAL_STORE + +#include "data_transformer.h" +#include "relational_db_sync_interface.h" +#include "sqlite_single_ver_continue_token.h" +#include "relational_schema_object.h" + +namespace DistributedDB { +struct ObjectData { +public: + void PutDataValue(const std::string &fieldName, const DataValue &value); + int GetDataValue(const std::string &fieldName, DataValue &value) const; +private: + mutable std::map fieldData; +}; + +struct VirtualRowData { + LogInfo logInfo; + ObjectData objectData; +}; + +class VirtualRelationalVerSyncDBInterface : public RelationalDBSyncInterface { +public: + VirtualRelationalVerSyncDBInterface() = default; + ~VirtualRelationalVerSyncDBInterface() override = default; + + int PutSyncDataWithQuery(const QueryObject &query, const std::vector &entries, + const std::string &deviceName) override; + + int PutLocalData(const std::vector &dataList, const std::string &tableName); + + RelationalSchemaObject GetSchemaInfo() const override; + + int GetDatabaseCreateTimestamp(Timestamp &outTime) const override; + + int GetBatchMetaData(const std::vector &keys, std::vector &entries) const override; + + int PutBatchMetaData(std::vector &entries) override; + + std::vector GetTablesQuery() override; + + int LocalDataChanged(int notifyEvent, std::vector &queryObj) override; + + int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const override; + + int GetInterfaceType() const override; + + void IncRefCount() override; + + void DecRefCount() override; + + std::vector GetIdentifier() const override; + + void GetMaxTimestamp(Timestamp &stamp) const override; + + // Get the max timestamp of one table. + int GetMaxTimestamp(const std::string &tableName, Timestamp ×tamp) const override; + + int GetMetaData(const Key &key, Value &value) const override; + + int PutMetaData(const Key &key, const Value &value) override; + + int DeleteMetaData(const std::vector &keys) override; + + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + + int GetAllMetaKeys(std::vector &keys) const override; + + const KvDBProperties &GetDbProperties() const override; + + void SetLocalFieldInfo(const std::vector &localFieldInfo); + + int GetAllSyncData(const std::string &tableName, std::vector &data); + + int GetVirtualSyncData(const std::string &tableName, const std::string &hashKey, VirtualRowData &data); + + int InterceptData(std::vector &entries, + const std::string &sourceID, const std::string &targetID) const override + { + return E_OK; + } + + int CheckAndInitQueryCondition(QueryObject &query) const override + { + return E_OK; + } + + int CreateDistributedDeviceTable(const std::string &device, const RelationalSyncStrategy &syncStrategy) override; + + int RegisterSchemaChangedCallback(const std::function &onSchemaChanged) override; + + void EraseSyncData(const std::string &tableName); + + void SetTableInfo(const TableInfo &tableInfo); + +private: + mutable std::map, std::vector> metadata_; + std::map> syncData_; + mutable std::map> localData_; + std::string schema_; + RelationalSchemaObject schemaObj_; + std::vector localFieldInfo_; + KvDBProperties properties_; + SecurityOption secOption_; +}; +} +#endif #endif // VIRTUAL_RELATIONAL_VER_SYNC_DB_INTERFACE_H \ No newline at end of file -- Gitee From 5919b2405c9eb5e199bc950ecc5159b4a2e81f22 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Fri, 1 Apr 2022 17:20:43 +0800 Subject: [PATCH 05/22] Reduce db cache lock scope Signed-off-by: lianhuix --- .../storage/src/relational_store_instance.cpp | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp index cb44f6fbb..443737900 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp @@ -79,6 +79,7 @@ static IRelationalStore *GetFromCache(const RelationalDBProperties &properties, { errCode = E_OK; std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); + std::lock_guard lockGuard(storeLock_); auto iter = dbs_.find(identifier); if (iter == dbs_.end()) { errCode = -E_NOT_FOUND; @@ -98,23 +99,17 @@ static IRelationalStore *GetFromCache(const RelationalDBProperties &properties, // Save to IKvDB to the global map void RelationalStoreInstance::RemoveKvDBFromCache(const RelationalDBProperties &properties) { - std::lock_guard lockGuard(storeLock_); std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); + std::lock_guard lockGuard(storeLock_); dbs_.erase(identifier); } void RelationalStoreInstance::SaveRelationalDBToCache(IRelationalStore *store, const RelationalDBProperties &properties) { - if (store == nullptr) { - return; - } - - { - std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); - store->WakeUpSyncer(); - if (dbs_.count(identifier) == 0) { - dbs_.insert(std::pair(identifier, store)); - } + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); + std::lock_guard lockGuard(storeLock_); + if (dbs_.count(identifier) == 0) { + dbs_.insert(std::pair(identifier, store)); } } @@ -138,6 +133,7 @@ IRelationalStore *RelationalStoreInstance::OpenDatabase(const RelationalDBProper RefObject::KillAndDecObjRef(db); return nullptr; } + db->WakeUpSyncer(); SaveRelationalDBToCache(db, properties); return db; @@ -145,7 +141,6 @@ IRelationalStore *RelationalStoreInstance::OpenDatabase(const RelationalDBProper IRelationalStore *RelationalStoreInstance::GetDataBase(const RelationalDBProperties &properties, int &errCode) { - std::lock_guard lockGuard(storeLock_); auto *db = GetFromCache(properties, errCode); if (db != nullptr) { LOGD("Get db from cache."); -- Gitee From 40378bbc70b486d1b150ebc230b986e9f67e2252 Mon Sep 17 00:00:00 2001 From: zqq Date: Tue, 29 Mar 2022 16:00:59 +0800 Subject: [PATCH 06/22] use connectionId to kill sync task Signed-off-by: zqq --- .../relational/sqlite_relational_store.cpp | 4 +-- .../relational/sqlite_relational_store.h | 2 +- .../sqlite_relational_store_connection.cpp | 4 +-- .../storage/src/sync_able_engine.cpp | 12 ++------ .../storage/src/sync_able_engine.h | 5 +--- .../storage/src/sync_able_kvdb.cpp | 4 +-- .../storage/src/sync_able_kvdb.h | 2 +- .../storage/src/sync_able_kvdb_connection.cpp | 3 +- .../distributeddb/syncer/include/isyncer.h | 2 +- .../syncer/include/syncer_proxy.h | 2 +- .../syncer/src/generic_syncer.cpp | 29 ++++++++++--------- .../distributeddb/syncer/src/generic_syncer.h | 7 +++-- .../src/single_ver_relational_syncer.cpp | 8 ++--- .../syncer/src/single_ver_relational_syncer.h | 6 ++-- .../distributeddb/syncer/src/syncer_proxy.cpp | 4 +-- 15 files changed, 44 insertions(+), 50 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index b50bc6bcc..f89342472 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -265,9 +265,9 @@ void SQLiteRelationalStore::ReleaseHandle(SQLiteSingleVerRelationalStorageExecut } } -int SQLiteRelationalStore::Sync(const ISyncer::SyncParma &syncParam) +int SQLiteRelationalStore::Sync(const ISyncer::SyncParma &syncParam, uint64_t connectionId) { - return syncAbleEngine_->Sync(syncParam); + return syncAbleEngine_->Sync(syncParam, connectionId); } // Called when a connection released. diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 15933421e..0557a1ca4 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -41,7 +41,7 @@ public: SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode) const; void ReleaseHandle(SQLiteSingleVerRelationalStorageExecutor *&handle) const; - int Sync(const ISyncer::SyncParma &syncParam); + int Sync(const ISyncer::SyncParma &syncParam, uint64_t connectionId); void ReleaseDBConnection(RelationalStoreConnection *connection); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index 41a30fabc..1a18d88ec 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -193,8 +193,8 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) syncParam.relationOnComplete = info.onComplete; syncParam.syncQuery = QuerySyncObject(info.query); syncParam.onFinalize = [this]() { DecObjRef(this); }; - - int errCode = store->Sync(syncParam); + uint64_t connectionId = 0; + int errCode = store->Sync(syncParam, connectionId); if (errCode != E_OK) { DecObjRef(this); return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp index 5c328dca7..229c544a0 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -30,7 +30,7 @@ SyncAbleEngine::~SyncAbleEngine() {} // Start a sync action. -int SyncAbleEngine::Sync(const ISyncer::SyncParma &parm) +int SyncAbleEngine::Sync(const ISyncer::SyncParma &parm, uint64_t connectionId) { if (!started_) { StartSyncer(); @@ -38,7 +38,7 @@ int SyncAbleEngine::Sync(const ISyncer::SyncParma &parm) return -E_NOT_INIT; } } - return syncer_.Sync(parm); + return syncer_.Sync(parm, connectionId); } void SyncAbleEngine::WakeUpSyncer() @@ -69,14 +69,6 @@ int SyncAbleEngine::DisableManualSync(void) return syncer_.DisableManualSync(); } -// Stop a sync action in progress. -void SyncAbleEngine::StopSync(int syncId) -{ - if (started_) { - syncer_.RemoveSyncOperation(syncId); - } -} - // Get The current virtual timestamp uint64_t SyncAbleEngine::GetTimestamp() { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h index 87afcc74c..d9b877f19 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h @@ -29,7 +29,7 @@ public: void TriggerSync(int notifyEvent); // Start a sync action. - int Sync(const ISyncer::SyncParma &parm); + int Sync(const ISyncer::SyncParma &parm, uint64_t connectionId); void WakeUpSyncer(); void Close(); @@ -40,9 +40,6 @@ public: int EnableManualSync(void); int DisableManualSync(void); - // Stop a sync action in progress. - void StopSync(int syncId); - // Get The current virtual timestamp uint64_t GetTimestamp(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp index c2c328675..5fb367bd4 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp @@ -78,7 +78,7 @@ void SyncAbleKvDB::Close() } // Start a sync action. -int SyncAbleKvDB::Sync(const ISyncer::SyncParma &parma) +int SyncAbleKvDB::Sync(const ISyncer::SyncParma &parma, uint64_t connectionId) { if (!started_) { int errCode = StartSyncer(); @@ -86,7 +86,7 @@ int SyncAbleKvDB::Sync(const ISyncer::SyncParma &parma) return errCode; } } - return syncer_.Sync(parma); + return syncer_.Sync(parma, connectionId); } void SyncAbleKvDB::EnableAutoSync(bool enable) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h index df6f2602a..aaeb723c2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h @@ -41,7 +41,7 @@ public: void Close() override; // Start a sync action. - int Sync(const ISyncer::SyncParma &parma); + int Sync(const ISyncer::SyncParma &parma, uint64_t connectionId); // Enable auto sync void EnableAutoSync(bool enable); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp index dbcf77fd9..3575b36c7 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp @@ -151,7 +151,8 @@ int SyncAbleKvDBConnection::PragmaSyncAction(const PragmaSync *syncParameter) syncParam.onFinalize = [this]() { DecObjRef(this); }; syncParam.onComplete = std::bind(&SyncAbleKvDBConnection::OnSyncComplete, this, std::placeholders::_1, syncParameter->onComplete_, syncParameter->wait_); - int errCode = kvDB->Sync(syncParam); + uint64_t connectionId = 0; + int errCode = kvDB->Sync(syncParam, connectionId); if (errCode != E_OK) { DecObjRef(this); } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h index 3fbc70fac..19be1d245 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h @@ -62,7 +62,7 @@ public: const std::function &onFinalize, bool wait) = 0; // Sync function. use SyncParma to reduce parameter. - virtual int Sync(const SyncParma ¶m) = 0; + virtual int Sync(const SyncParma ¶m, uint64_t connectionId) = 0; // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. virtual int RemoveSyncOperation(int syncId) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h index 8a1c8c279..5e7c28e18 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h @@ -46,7 +46,7 @@ public: const std::function &onFinalize, bool wait) override; // Sync function. use SyncParma to reduce parameter. - int Sync(const SyncParma ¶m) override; + int Sync(const SyncParma ¶m, uint64_t connectionId) override; // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. int RemoveSyncOperation(int syncId) override; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp index 83dd6c1cc..d76d4e758 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp @@ -183,7 +183,7 @@ int GenericSyncer::Sync(const InternalSyncParma ¶m) return Sync(syncParam); } -int GenericSyncer::Sync(const SyncParma ¶m) +int GenericSyncer::Sync(const SyncParma ¶m, uint64_t connectionId) { int errCode = SyncParamCheck(param); if (errCode != E_OK) { @@ -195,7 +195,7 @@ int GenericSyncer::Sync(const SyncParma ¶m) } uint32_t syncId = GenerateSyncId(); - errCode = PrepareSync(param, syncId); + errCode = PrepareSync(param, syncId, connectionId); if (errCode != E_OK) { LOGE("[Syncer] PrepareSync failed when sync called, err %d", errCode); return errCode; @@ -204,7 +204,7 @@ int GenericSyncer::Sync(const SyncParma ¶m) return E_OK; } -int GenericSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) +int GenericSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId, uint64_t connectionId) { auto *operation = new (std::nothrow) SyncOperation(syncId, param.devices, param.mode, param.onComplete, param.wait); @@ -217,14 +217,15 @@ int GenericSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) std::lock_guard autoLock(syncerLock_); PerformanceAnalysis::GetInstance()->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); InitSyncOperation(operation, param); - LOGI("[Syncer] GenerateSyncId %d, mode = %d, wait = %d , label = %s, devices = %s", syncId, param.mode, - param.wait, label_.c_str(), GetSyncDevicesStr(param.devices).c_str()); + LOGI("[Syncer] GenerateSyncId %" PRIu32 ", mode = %d, wait = %d, label = %s, devices = %s", syncId, param.mode, + param.wait, label_.c_str(), GetSyncDevicesStr(param.devices).c_str()); AddSyncOperation(operation); PerformanceAnalysis::GetInstance()->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); } if (!param.wait) { std::lock_guard lockGuard(syncIdLock_); - syncIdList_.push_back(static_cast(syncId)); + connectionIdMap_[connectionId].push_back(static_cast(syncId)); + syncIdMap_[static_cast(syncId)] = connectionId; } if (operation->CheckIsAllFinished()) { operation->Finished(); @@ -253,7 +254,11 @@ int GenericSyncer::RemoveSyncOperation(int syncId) RefObject::KillAndDecObjRef(operation); operation = nullptr; std::lock_guard lockGuard(syncIdLock_); - syncIdList_.remove(syncId); + uint64_t connectionId = syncIdMap_[syncId]; + if (connectionIdMap_.find(connectionId) != connectionIdMap_.end()) { + connectionIdMap_[connectionId].remove(syncId); + } + syncIdMap_.erase(syncId); return E_OK; } return -E_INVALID_ARGS; @@ -261,12 +266,14 @@ int GenericSyncer::RemoveSyncOperation(int syncId) int GenericSyncer::StopSync() { + uint64_t connectionId = 0; std::list syncIdList; { std::lock_guard lockGuard(syncIdLock_); - syncIdList = syncIdList_; + syncIdList = connectionIdMap_[connectionId]; + connectionIdMap_.erase(connectionId); } - for (const auto &syncId : syncIdList) { + for (auto syncId : syncIdList) { RemoveSyncOperation(syncId); } return E_OK; @@ -474,10 +481,6 @@ void GenericSyncer::TriggerSyncFinished(SyncOperation *operation) void GenericSyncer::OnSyncFinished(int syncId) { - { - std::lock_guard lockGuard(syncIdLock_); - syncIdList_.remove(syncId); - } (void)(RemoveSyncOperation(syncId)); } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h index e675d4d9e..f30db53ef 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h @@ -51,7 +51,7 @@ public: const std::function &onFinalize, bool wait) override; // Sync function. use SyncParma to reduce parameter. - int Sync(const SyncParma ¶m) override; + int Sync(const SyncParma ¶m, uint64_t connectionId = 0) override; // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. int RemoveSyncOperation(int syncId) override; @@ -103,7 +103,7 @@ protected: // Create a sync engine, if has memory error, will return nullptr. virtual ISyncEngine *CreateSyncEngine() = 0; - virtual int PrepareSync(const SyncParma ¶m, uint32_t syncId); + virtual int PrepareSync(const SyncParma ¶m, uint32_t syncId, uint64_t connectionId); // Add a Sync Operation, after call this function, the operation will be start virtual void AddSyncOperation(SyncOperation *operation); @@ -178,7 +178,8 @@ protected: static int currentSyncId_; static std::mutex syncIdLock_; // For sync in progress. - std::list syncIdList_; + std::map> connectionIdMap_; + std::map syncIdMap_; ISyncEngine *syncEngine_; ISyncInterface *syncInterface_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp index 08e87474d..6436720bd 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp @@ -41,7 +41,7 @@ int SingleVerRelationalSyncer::Sync(const SyncParma ¶m) return GenericSyncer::Sync(param); } -int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) +int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId, uint64_t connectionId) { const auto &syncInterface = static_cast(syncInterface_); std::vector tablesQuery; @@ -51,7 +51,7 @@ int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t sync tablesQuery = syncInterface->GetTablesQuery(); } std::set subSyncIdSet; - int errCode = GenerateEachSyncTask(param, syncId, tablesQuery, subSyncIdSet); + int errCode = GenerateEachSyncTask(param, syncId, tablesQuery, connectionId, subSyncIdSet); if (errCode != E_OK) { DoRollBack(subSyncIdSet); return errCode; @@ -63,7 +63,7 @@ int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t sync } int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, - const std::vector &tablesQuery, std::set &subSyncIdSet) + const std::vector &tablesQuery, uint64_t connectionId, std::set &subSyncIdSet) { SyncParma subParam = param; subParam.isQuerySync = true; @@ -80,7 +80,7 @@ int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint std::lock_guard lockGuard(syncMapLock_); fullSyncIdMap_[syncId].insert(subSyncId); } - errCode = GenericSyncer::PrepareSync(subParam, subSyncId); + errCode = GenericSyncer::PrepareSync(subParam, subSyncId, connectionId); if (errCode != E_OK) { LOGW("[SingleVerRelationalSyncer] PrepareSync failed errCode:%d", errCode); std::lock_guard lockGuard(syncMapLock_); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h index 9360e0f3a..5bb7caee0 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h @@ -25,7 +25,7 @@ public: int Initialize(ISyncInterface *syncInterface, bool isNeedActive) override; // Sync function. use SyncParma to reduce parameter. - int Sync(const SyncParma ¶m) override; + int Sync(const SyncParma ¶m, uint64_t connectionId) override; void EnableAutoSync(bool enable) override; @@ -33,7 +33,7 @@ public: protected: - int PrepareSync(const SyncParma ¶m, uint32_t syncId) override; + int PrepareSync(const SyncParma ¶m, uint32_t syncId, uint64_t connectionId) override; int SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, const std::vector &devices) const override; @@ -41,7 +41,7 @@ protected: private: int GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, - const std::vector &tablesQuery, std::set &subSyncIdSet); + const std::vector &tablesQuery, uint64_t connectionId, std::set &subSyncIdSet); void DoRollBack(std::set &subSyncIdSet); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp index 0c71b0da4..8c759cd0b 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp @@ -64,12 +64,12 @@ int SyncerProxy::Sync(const std::vector &devices, int mode, return syncer_->Sync(devices, mode, onComplete, onFinalize, wait); } -int SyncerProxy::Sync(const SyncParma &parma) +int SyncerProxy::Sync(const SyncParma &parma, uint64_t connectionId) { if (syncer_ == nullptr) { return -E_NOT_INIT; } - return syncer_->Sync(parma); + return syncer_->Sync(parma, connectionId); } int SyncerProxy::RemoveSyncOperation(int syncId) -- Gitee From e318e298214b2275bf58f5ba3daa5c10dc35e21f Mon Sep 17 00:00:00 2001 From: zqq Date: Tue, 29 Mar 2022 17:03:45 +0800 Subject: [PATCH 07/22] kill sync task when close connection Signed-off-by: zqq --- .../common/include/runtime_context.h | 3 +++ .../common/src/runtime_context_impl.cpp | 10 +++++++- .../common/src/runtime_context_impl.h | 5 ++++ .../relational/sqlite_relational_store.cpp | 5 ++++ .../relational/sqlite_relational_store.h | 2 ++ .../sqlite_relational_store_connection.cpp | 25 ++++++++++++++++--- .../sqlite_relational_store_connection.h | 5 ++++ .../storage/src/sync_able_engine.cpp | 7 ++++++ .../storage/src/sync_able_engine.h | 2 ++ .../storage/src/sync_able_kvdb.cpp | 4 +-- .../storage/src/sync_able_kvdb.h | 2 +- .../storage/src/sync_able_kvdb_connection.cpp | 15 +++++++++-- .../storage/src/sync_able_kvdb_connection.h | 5 ++++ .../distributeddb/syncer/include/isyncer.h | 2 +- .../syncer/include/syncer_proxy.h | 2 +- .../syncer/src/generic_syncer.cpp | 3 +-- .../distributeddb/syncer/src/generic_syncer.h | 2 +- .../distributeddb/syncer/src/syncer_proxy.cpp | 4 +-- 18 files changed, 87 insertions(+), 16 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h index 9b68f63f6..01f79c369 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h @@ -123,6 +123,9 @@ public: EventType event) = 0; virtual int NotifyUserChanged() const = 0; + + // Generate global sessionId in current process + virtual uint64_t GenerateSessionId() = 0; protected: RuntimeContext() = default; virtual ~RuntimeContext() {} diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp index 6c95824e3..a54033f50 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp @@ -29,7 +29,8 @@ RuntimeContextImpl::RuntimeContextImpl() taskPoolReportsTimerId_(0), timeTickMonitor_(nullptr), systemApiAdapter_(nullptr), - lockStatusObserver_(nullptr) + lockStatusObserver_(nullptr), + lastSessionId_(0) { } @@ -654,4 +655,11 @@ int RuntimeContextImpl::NotifyUserChanged() const userChangeMonitor_->NotifyUserChanged(); return E_OK; } + +uint64_t RuntimeContextImpl::GenerateSessionId() +{ + std::lock_guard autoLock(sessionIdLock_); + lastSessionId_++; + return lastSessionId_; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h index 5dd607e84..8875efb9f 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h @@ -113,6 +113,8 @@ public: EventType event) override; // Notify TIME_CHANGE_EVENT. int NotifyUserChanged() const override; + + uint64_t GenerateSessionId() override; private: static constexpr int MAX_TP_THREADS = 10; // max threads of the task pool. static constexpr int MIN_TP_THREADS = 1; // min threads of the task pool. @@ -167,6 +169,9 @@ private: mutable std::mutex userChangeMonitorLock_; std::unique_ptr userChangeMonitor_; + + mutable std::mutex sessionIdLock_; + uint64_t lastSessionId_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index f89342472..6708d9a5a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -490,5 +490,10 @@ RelationalDBProperties SQLiteRelationalStore::GetProperties() const { return properties_; } + +void SQLiteRelationalStore::StopSync(uint64_t connectionId) +{ + return syncAbleEngine_->StopSync(connectionId); +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 0557a1ca4..2811a6cbe 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -64,6 +64,8 @@ public: RelationalDBProperties GetProperties() const override; + void StopSync(uint64_t connectionId); + private: void ReleaseResources(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index 1a18d88ec..31ed41255 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -19,7 +19,18 @@ namespace DistributedDB { SQLiteRelationalStoreConnection::SQLiteRelationalStoreConnection(SQLiteRelationalStore *store) - : RelationalStoreConnection(store) {} + : RelationalStoreConnection(store), connectionId_(0) +{ + OnKill([this]() { + auto *store = GetDB(); + if (store == nullptr) { + return; + } + UnlockObj(); + store->StopSync(GetConnectionId()); + LockObj(); + }); +} // Close and release the connection. int SQLiteRelationalStoreConnection::Close() { @@ -193,8 +204,7 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) syncParam.relationOnComplete = info.onComplete; syncParam.syncQuery = QuerySyncObject(info.query); syncParam.onFinalize = [this]() { DecObjRef(this); }; - uint64_t connectionId = 0; - int errCode = store->Sync(syncParam, connectionId); + int errCode = store->Sync(syncParam, GetConnectionId()); if (errCode != E_OK) { DecObjRef(this); return errCode; @@ -218,5 +228,14 @@ void SQLiteRelationalStoreConnection::RegisterObserverAction(const RelationalObs { static_cast(store_)->RegisterObserverAction(action); } + +uint64_t SQLiteRelationalStoreConnection::GetConnectionId() +{ + std::mutex connectionIdLock_; + if (connectionId_ == 0) { + connectionId_ = RuntimeContext::GetInstance()->GenerateSessionId(); + } + return connectionId_; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 13b2dbd26..d68b75c1c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -58,8 +58,13 @@ private: // Roll back the transaction int RollBack(); + uint64_t GetConnectionId(); + SQLiteSingleVerRelationalStorageExecutor *writeHandle_ = nullptr; mutable std::mutex transactionMutex_; // used for transaction + + std::mutex connectionIdLock_; + uint64_t connectionId_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp index 229c544a0..79b20f13c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -133,4 +133,11 @@ int SyncAbleEngine::GetLocalIdentity(std::string &outTarget) } return syncer_.GetLocalIdentity(outTarget); } + +void SyncAbleEngine::StopSync(uint64_t connectionId) +{ + if (started_) { + syncer_.StopSync(connectionId); + } +} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h index d9b877f19..c1fc4e8bd 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h @@ -47,6 +47,8 @@ public: int GetLocalIdentity(std::string &outTarget); + // Stop a sync action in progress + void StopSync(uint64_t connectionId); private: // Start syncer void StartSyncer(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp index 5fb367bd4..76b67c97a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp @@ -103,10 +103,10 @@ void SyncAbleKvDB::WakeUpSyncer() } // Stop a sync action in progress. -void SyncAbleKvDB::StopSync() +void SyncAbleKvDB::StopSync(uint64_t connectionId) { if (started_) { - syncer_.StopSync(); + syncer_.StopSync(connectionId); } } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h index aaeb723c2..91a8d6c8d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h @@ -47,7 +47,7 @@ public: void EnableAutoSync(bool enable); // Stop a sync action in progress. - void StopSync(); + void StopSync(uint64_t connectionId); // Get The current virtual timestamp uint64_t GetTimestamp(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp index 3575b36c7..25dd8e42e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp @@ -20,12 +20,14 @@ #include "db_constant.h" #include "kvdb_pragma.h" #include "performance_analysis.h" +#include "runtime_context.h" #include "sync_able_kvdb.h" namespace DistributedDB { SyncAbleKvDBConnection::SyncAbleKvDBConnection(SyncAbleKvDB *kvDB) : GenericKvDBConnection(kvDB), - remotePushFinishedListener_(nullptr) + remotePushFinishedListener_(nullptr), + connectionId_(0) { OnKill([this]() { auto *db = GetDB(); @@ -34,7 +36,7 @@ SyncAbleKvDBConnection::SyncAbleKvDBConnection(SyncAbleKvDB *kvDB) } // Drop the lock before we call RemoveSyncOperation(). UnlockObj(); - db->StopSync(); + db->StopSync(GetConnectionId()); LockObj(); }); } @@ -323,4 +325,13 @@ int SyncAbleKvDBConnection::SetPushDataInterceptor(const PushDataInterceptor &in kvDB->SetDataInterceptor(interceptor); return E_OK; } + +uint64_t SyncAbleKvDBConnection::GetConnectionId() +{ + std::lock_guard autoLock(connectionLock_); + if (connectionId_ == 0) { + connectionId_ = RuntimeContext::GetInstance()->GenerateSessionId(); + } + return connectionId_; +} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h index a640bbe0f..e90bdf765 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h @@ -68,9 +68,14 @@ private: int SetPushDataInterceptor(const PushDataInterceptor &interceptor); + uint64_t GetConnectionId(); + std::mutex remotePushFinishedListenerLock_; NotificationChain::Listener *remotePushFinishedListener_; std::map> pragmaFunc_{}; + + std::mutex connectionLock_; + uint64_t connectionId_; }; } #endif // SYNC_ABLE_KVDB_CONNECTION_H diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h index 19be1d245..e02221a65 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h @@ -67,7 +67,7 @@ public: // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. virtual int RemoveSyncOperation(int syncId) = 0; - virtual int StopSync() = 0; + virtual int StopSync(uint64_t connectionId) = 0; // Get The current virtual timestamp virtual uint64_t GetTimestamp() = 0; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h index 5e7c28e18..6ef417b79 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h @@ -51,7 +51,7 @@ public: // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. int RemoveSyncOperation(int syncId) override; - int StopSync() override; + int StopSync(uint64_t connectionId) override; // Get The current virtual timestamp uint64_t GetTimestamp() override; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp index d76d4e758..9697654f9 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp @@ -264,9 +264,8 @@ int GenericSyncer::RemoveSyncOperation(int syncId) return -E_INVALID_ARGS; } -int GenericSyncer::StopSync() +int GenericSyncer::StopSync(uint64_t connectionId) { - uint64_t connectionId = 0; std::list syncIdList; { std::lock_guard lockGuard(syncIdLock_); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h index f30db53ef..60bc32a56 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h @@ -56,7 +56,7 @@ public: // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. int RemoveSyncOperation(int syncId) override; - int StopSync() override; + int StopSync(uint64_t connectionId) override; // Get The current virtual timestamp uint64_t GetTimestamp() override; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp index 8c759cd0b..6649beb43 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp @@ -80,12 +80,12 @@ int SyncerProxy::RemoveSyncOperation(int syncId) return syncer_->RemoveSyncOperation(syncId); } -int SyncerProxy::StopSync() +int SyncerProxy::StopSync(uint64_t connectionId) { if (syncer_ == nullptr) { return -E_NOT_INIT; } - return syncer_->StopSync(); + return syncer_->StopSync(connectionId); } uint64_t SyncerProxy::GetTimestamp() -- Gitee From e72ce7aff98da3fd78dae448368de056746ec6fe Mon Sep 17 00:00:00 2001 From: zqq Date: Thu, 31 Mar 2022 11:32:52 +0800 Subject: [PATCH 08/22] move GetConnectionId func to iconnection Signed-off-by: zqq --- .../libs/distributeddb/BUILD.gn | 1 + .../storage/include/iconnection.h | 40 +++++++++++++++++++ .../storage/include/ikvdb_connection.h | 5 ++- .../include/relational_store_connection.h | 3 +- .../distributeddb/storage/src/iconnection.cpp | 33 +++++++++++++++ .../sqlite_relational_store_connection.cpp | 11 +---- .../sqlite_relational_store_connection.h | 3 -- .../storage/src/sync_able_kvdb_connection.cpp | 12 +----- .../storage/src/sync_able_kvdb_connection.h | 3 -- .../libs/distributeddb/test/BUILD.gn | 1 + 10 files changed, 82 insertions(+), 30 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index 55c3dcd34..0198fa1cc 100644 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -136,6 +136,7 @@ ohos_shared_library("distributeddb") { "storage/src/generic_kvdb.cpp", "storage/src/generic_kvdb_connection.cpp", "storage/src/generic_single_ver_kv_entry.cpp", + "storage/src/iconnection.cpp", "storage/src/ikvdb_factory.cpp", "storage/src/kvdb_commit_notify_filterable_data.cpp", "storage/src/kvdb_manager.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h new file mode 100644 index 000000000..79e84e0b1 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef I_CONNECTION_H +#define I_CONNECTION_H + +#include + +#include "macro_utils.h" + +namespace DistributedDB { + +class IConnection { +public: + IConnection(); + virtual ~IConnection() {}; + + DISABLE_COPY_ASSIGN_MOVE(IConnection); + +protected: + uint64_t GetConnectionId(); + + std::mutex connectionLock_; + uint64_t connectionId_; +}; +} // namespace DistributedDB + +#endif // I_KV_DB_CONNECTION_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h index 09f0e5ec7..b051f95bd 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h @@ -19,10 +19,11 @@ #include #include -#include "store_types.h" #include "db_types.h" +#include "iconnection.h" #include "macro_utils.h" #include "query.h" +#include "store_types.h" namespace DistributedDB { class IKvDB; @@ -34,7 +35,7 @@ class IKvDBResultSet; using KvDBObserverAction = std::function; using KvDBConflictAction = std::function; -class IKvDBConnection { +class IKvDBConnection : public IConnection { public: IKvDBConnection() = default; virtual ~IKvDBConnection() {}; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index be53b2c6c..4c9ab7b6d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -20,6 +20,7 @@ #include #include "db_types.h" +#include "iconnection.h" #include "macro_utils.h" #include "ref_object.h" #include "relational_store_delegate.h" @@ -27,7 +28,7 @@ namespace DistributedDB { class IRelationalStore; using RelationalObserverAction = std::function; -class RelationalStoreConnection : public virtual RefObject { +class RelationalStoreConnection : public IConnection, public virtual RefObject { public: struct SyncInfo { const std::vector &devices; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp new file mode 100644 index 000000000..dc2d8d1c1 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "iconnection.h" + +#include "runtime_context.h" + +namespace DistributedDB { +IConnection::IConnection() + : connectionId_(0) +{ +} + +uint64_t IConnection::GetConnectionId() +{ + std::mutex connectionIdLock_; + if (connectionId_ == 0) { + connectionId_ = RuntimeContext::GetInstance()->GenerateSessionId(); + } + return connectionId_; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index 31ed41255..0628f32ef 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -19,7 +19,7 @@ namespace DistributedDB { SQLiteRelationalStoreConnection::SQLiteRelationalStoreConnection(SQLiteRelationalStore *store) - : RelationalStoreConnection(store), connectionId_(0) + : RelationalStoreConnection(store) { OnKill([this]() { auto *store = GetDB(); @@ -228,14 +228,5 @@ void SQLiteRelationalStoreConnection::RegisterObserverAction(const RelationalObs { static_cast(store_)->RegisterObserverAction(action); } - -uint64_t SQLiteRelationalStoreConnection::GetConnectionId() -{ - std::mutex connectionIdLock_; - if (connectionId_ == 0) { - connectionId_ = RuntimeContext::GetInstance()->GenerateSessionId(); - } - return connectionId_; -} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index d68b75c1c..c68f1f70b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -62,9 +62,6 @@ private: SQLiteSingleVerRelationalStorageExecutor *writeHandle_ = nullptr; mutable std::mutex transactionMutex_; // used for transaction - - std::mutex connectionIdLock_; - uint64_t connectionId_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp index 25dd8e42e..9faba5193 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp @@ -26,8 +26,7 @@ namespace DistributedDB { SyncAbleKvDBConnection::SyncAbleKvDBConnection(SyncAbleKvDB *kvDB) : GenericKvDBConnection(kvDB), - remotePushFinishedListener_(nullptr), - connectionId_(0) + remotePushFinishedListener_(nullptr) { OnKill([this]() { auto *db = GetDB(); @@ -325,13 +324,4 @@ int SyncAbleKvDBConnection::SetPushDataInterceptor(const PushDataInterceptor &in kvDB->SetDataInterceptor(interceptor); return E_OK; } - -uint64_t SyncAbleKvDBConnection::GetConnectionId() -{ - std::lock_guard autoLock(connectionLock_); - if (connectionId_ == 0) { - connectionId_ = RuntimeContext::GetInstance()->GenerateSessionId(); - } - return connectionId_; -} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h index e90bdf765..23f142a69 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h @@ -73,9 +73,6 @@ private: std::mutex remotePushFinishedListenerLock_; NotificationChain::Listener *remotePushFinishedListener_; std::map> pragmaFunc_{}; - - std::mutex connectionLock_; - uint64_t connectionId_; }; } #endif // SYNC_ABLE_KVDB_CONNECTION_H diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 4e3d131ca..aee25ed1a 100644 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -139,6 +139,7 @@ ohos_source_set("src_file") { "../storage/src/generic_kvdb.cpp", "../storage/src/generic_kvdb_connection.cpp", "../storage/src/generic_single_ver_kv_entry.cpp", + "../storage/src/iconnection.cpp", "../storage/src/ikvdb_factory.cpp", "../storage/src/kvdb_commit_notify_filterable_data.cpp", "../storage/src/kvdb_manager.cpp", -- Gitee From a6192554757abd62d835f193e20d186f126bc46d Mon Sep 17 00:00:00 2001 From: zqq Date: Thu, 31 Mar 2022 11:34:49 +0800 Subject: [PATCH 09/22] remove mutable Signed-off-by: zqq --- .../common/src/runtime_context_impl.h | 2 +- .../storage/include/iconnection.h | 2 ++ .../distributeddb/storage/src/iconnection.cpp | 4 ++-- .../sqlite_relational_store_connection.h | 2 -- .../storage/src/sync_able_kvdb_connection.cpp | 3 +-- .../storage/src/sync_able_kvdb_connection.h | 2 -- .../syncer/src/generic_syncer.cpp | 18 +++++++++++++++++- .../distributeddb/syncer/src/generic_syncer.h | 4 +++- .../src/single_ver_relational_syncer.cpp | 4 ++-- 9 files changed, 28 insertions(+), 13 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h index 8875efb9f..a6277d66b 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h @@ -170,7 +170,7 @@ private: mutable std::mutex userChangeMonitorLock_; std::unique_ptr userChangeMonitor_; - mutable std::mutex sessionIdLock_; + std::mutex sessionIdLock_; uint64_t lastSessionId_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h index 79e84e0b1..faf45953c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h @@ -29,6 +29,8 @@ public: DISABLE_COPY_ASSIGN_MOVE(IConnection); + static constexpr uint64_t INVALID_CONNECTION_ID = 0; + protected: uint64_t GetConnectionId(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp index dc2d8d1c1..270d3f23b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp @@ -18,14 +18,14 @@ namespace DistributedDB { IConnection::IConnection() - : connectionId_(0) + : connectionId_(INVALID_CONNECTION_ID) { } uint64_t IConnection::GetConnectionId() { std::mutex connectionIdLock_; - if (connectionId_ == 0) { + if (connectionId_ == INVALID_CONNECTION_ID) { connectionId_ = RuntimeContext::GetInstance()->GenerateSessionId(); } return connectionId_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index c68f1f70b..13b2dbd26 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -58,8 +58,6 @@ private: // Roll back the transaction int RollBack(); - uint64_t GetConnectionId(); - SQLiteSingleVerRelationalStorageExecutor *writeHandle_ = nullptr; mutable std::mutex transactionMutex_; // used for transaction }; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp index 9faba5193..ca10da6db 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp @@ -152,8 +152,7 @@ int SyncAbleKvDBConnection::PragmaSyncAction(const PragmaSync *syncParameter) syncParam.onFinalize = [this]() { DecObjRef(this); }; syncParam.onComplete = std::bind(&SyncAbleKvDBConnection::OnSyncComplete, this, std::placeholders::_1, syncParameter->onComplete_, syncParameter->wait_); - uint64_t connectionId = 0; - int errCode = kvDB->Sync(syncParam, connectionId); + int errCode = kvDB->Sync(syncParam, GetConnectionId()); if (errCode != E_OK) { DecObjRef(this); } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h index 23f142a69..a640bbe0f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h @@ -68,8 +68,6 @@ private: int SetPushDataInterceptor(const PushDataInterceptor &interceptor); - uint64_t GetConnectionId(); - std::mutex remotePushFinishedListenerLock_; NotificationChain::Listener *remotePushFinishedListener_; std::map> pragmaFunc_{}; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp index 9697654f9..a5a567106 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp @@ -183,6 +183,11 @@ int GenericSyncer::Sync(const InternalSyncParma ¶m) return Sync(syncParam); } +int GenericSyncer::Sync(const SyncParma ¶m) +{ + return Sync(param, IConnection::INVALID_CONNECTION_ID); +} + int GenericSyncer::Sync(const SyncParma ¶m, uint64_t connectionId) { int errCode = SyncParamCheck(param); @@ -222,7 +227,7 @@ int GenericSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId, uint64_t AddSyncOperation(operation); PerformanceAnalysis::GetInstance()->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); } - if (!param.wait) { + if (!param.wait && connectionId != IConnection::INVALID_CONNECTION_ID) { std::lock_guard lockGuard(syncIdLock_); connectionIdMap_[connectionId].push_back(static_cast(syncId)); syncIdMap_[static_cast(syncId)] = connectionId; @@ -254,6 +259,9 @@ int GenericSyncer::RemoveSyncOperation(int syncId) RefObject::KillAndDecObjRef(operation); operation = nullptr; std::lock_guard lockGuard(syncIdLock_); + if (syncIdMap_.find(syncId) == syncIdMap_.end()) { + return E_OK; + } uint64_t connectionId = syncIdMap_[syncId]; if (connectionIdMap_.find(connectionId) != connectionIdMap_.end()) { connectionIdMap_[connectionId].remove(syncId); @@ -269,6 +277,9 @@ int GenericSyncer::StopSync(uint64_t connectionId) std::list syncIdList; { std::lock_guard lockGuard(syncIdLock_); + if (connectionIdMap_.find(connectionId) == connectionIdMap_.end()) { + return E_OK; + } syncIdList = connectionIdMap_[connectionId]; connectionIdMap_.erase(connectionId); } @@ -469,6 +480,11 @@ void GenericSyncer::ClearSyncOperations(bool isClosedOperation) } syncOperationMap_.clear(); } + { + std::lock_guard lock(syncIdLock_); + connectionIdMap_.clear(); + syncIdMap_.clear(); + } } void GenericSyncer::TriggerSyncFinished(SyncOperation *operation) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h index 60bc32a56..9ccfad58f 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h @@ -51,7 +51,9 @@ public: const std::function &onFinalize, bool wait) override; // Sync function. use SyncParma to reduce parameter. - int Sync(const SyncParma ¶m, uint64_t connectionId = 0) override; + int Sync(const SyncParma ¶m); + + int Sync(const SyncParma ¶m, uint64_t connectionId) override; // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. int RemoveSyncOperation(int syncId) override; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp index 6436720bd..264ab16c0 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp @@ -30,7 +30,7 @@ int SingleVerRelationalSyncer::Initialize(ISyncInterface *syncInterface, bool is RegisterSchemaChangedCallback(callback); } -int SingleVerRelationalSyncer::Sync(const SyncParma ¶m) +int SingleVerRelationalSyncer::Sync(const SyncParma ¶m, uint64_t connectionId) { if (param.mode == SYNC_MODE_PUSH_PULL) { return -E_NOT_SUPPORT; @@ -38,7 +38,7 @@ int SingleVerRelationalSyncer::Sync(const SyncParma ¶m) if (param.syncQuery.GetRelationTableName().empty()) { return -E_NOT_SUPPORT; } - return GenericSyncer::Sync(param); + return GenericSyncer::Sync(param, connectionId); } int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId, uint64_t connectionId) -- Gitee From f849435a8e5ea6a7e107d7e20a0e08b8d1128e65 Mon Sep 17 00:00:00 2001 From: zqq Date: Thu, 31 Mar 2022 15:56:47 +0800 Subject: [PATCH 10/22] rename INVALID_CONNECTION_ID to IGNORE_CONNECTION_ID Signed-off-by: zqq --- .../libs/distributeddb/common/include/db_constant.h | 1 + .../libs/distributeddb/storage/include/iconnection.h | 4 +--- .../libs/distributeddb/storage/src/iconnection.cpp | 6 +++--- .../libs/distributeddb/syncer/src/generic_syncer.cpp | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index a94051de6..6ecd3149f 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -130,6 +130,7 @@ public: static constexpr int RELATIONAL_LOG_TABLE_FIELD_NUM = 7; // field num is relational distributed log table + static constexpr uint64_t IGNORE_CONNECTION_ID = 0; // For relational static const std::string RELATIONAL_PREFIX; static const std::string TIMESTAMP_ALIAS; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h index faf45953c..220dd58ce 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h @@ -29,12 +29,10 @@ public: DISABLE_COPY_ASSIGN_MOVE(IConnection); - static constexpr uint64_t INVALID_CONNECTION_ID = 0; - protected: uint64_t GetConnectionId(); - std::mutex connectionLock_; + std::mutex connectionIdLock_; uint64_t connectionId_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp index 270d3f23b..b03da031f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp @@ -18,14 +18,14 @@ namespace DistributedDB { IConnection::IConnection() - : connectionId_(INVALID_CONNECTION_ID) + : connectionId_(0) { } uint64_t IConnection::GetConnectionId() { - std::mutex connectionIdLock_; - if (connectionId_ == INVALID_CONNECTION_ID) { + std::lock_guard autoLock(connectionIdLock_); + if (connectionId_ == 0) { connectionId_ = RuntimeContext::GetInstance()->GenerateSessionId(); } return connectionId_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp index a5a567106..6450c1bdc 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp @@ -185,7 +185,7 @@ int GenericSyncer::Sync(const InternalSyncParma ¶m) int GenericSyncer::Sync(const SyncParma ¶m) { - return Sync(param, IConnection::INVALID_CONNECTION_ID); + return Sync(param, DBConstant::IGNORE_CONNECTION_ID); } int GenericSyncer::Sync(const SyncParma ¶m, uint64_t connectionId) @@ -227,7 +227,7 @@ int GenericSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId, uint64_t AddSyncOperation(operation); PerformanceAnalysis::GetInstance()->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); } - if (!param.wait && connectionId != IConnection::INVALID_CONNECTION_ID) { + if (!param.wait && connectionId != DBConstant::IGNORE_CONNECTION_ID) { std::lock_guard lockGuard(syncIdLock_); connectionIdMap_[connectionId].push_back(static_cast(syncId)); syncIdMap_[static_cast(syncId)] = connectionId; -- Gitee From 1fa59e5692a64e6760330b655f49355fff3410fb Mon Sep 17 00:00:00 2001 From: zqq Date: Thu, 14 Apr 2022 11:13:37 +0800 Subject: [PATCH 11/22] use atomic replace mutex Signed-off-by: zqq --- .../libs/distributeddb/common/src/runtime_context_impl.cpp | 6 ++---- .../libs/distributeddb/common/src/runtime_context_impl.h | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp index a54033f50..9c02425c1 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp @@ -30,7 +30,7 @@ RuntimeContextImpl::RuntimeContextImpl() timeTickMonitor_(nullptr), systemApiAdapter_(nullptr), lockStatusObserver_(nullptr), - lastSessionId_(0) + currentSessionId_(1) { } @@ -658,8 +658,6 @@ int RuntimeContextImpl::NotifyUserChanged() const uint64_t RuntimeContextImpl::GenerateSessionId() { - std::lock_guard autoLock(sessionIdLock_); - lastSessionId_++; - return lastSessionId_; + return currentSessionId_.fetch_add(1, std::memory_order_seq_cst); } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h index a6277d66b..b998b9da5 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h @@ -170,8 +170,7 @@ private: mutable std::mutex userChangeMonitorLock_; std::unique_ptr userChangeMonitor_; - std::mutex sessionIdLock_; - uint64_t lastSessionId_; + std::atomic currentSessionId_; }; } // namespace DistributedDB -- Gitee From f8431dcf6373ffc7e82866ed0b5c9ec0bf8a0133 Mon Sep 17 00:00:00 2001 From: zqq Date: Thu, 14 Apr 2022 14:32:34 +0800 Subject: [PATCH 12/22] use atomic in iconnection Signed-off-by: zqq --- .../libs/distributeddb/storage/include/iconnection.h | 3 ++- .../libs/distributeddb/storage/src/iconnection.cpp | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h index 220dd58ce..e2e150f31 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h @@ -16,6 +16,7 @@ #ifndef I_CONNECTION_H #define I_CONNECTION_H +#include #include #include "macro_utils.h" @@ -33,7 +34,7 @@ protected: uint64_t GetConnectionId(); std::mutex connectionIdLock_; - uint64_t connectionId_; + std::atomic connectionId_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp index b03da031f..478397e2a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp @@ -24,7 +24,11 @@ IConnection::IConnection() uint64_t IConnection::GetConnectionId() { + if (connectionId_ != 0) { + return connectionId_; + } std::lock_guard autoLock(connectionIdLock_); + // check again here, may be generated after get lock if (connectionId_ == 0) { connectionId_ = RuntimeContext::GetInstance()->GenerateSessionId(); } -- Gitee From dc118e3a5224fdd0870f1ded7ce9389f23b0af17 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Sat, 16 Apr 2022 17:37:22 +0800 Subject: [PATCH 13/22] Fix codecheck Signed-off-by: lianhuix --- .../storage/include/iconnection.h | 1 - .../src/relational_sync_able_storage.cpp | 2 +- .../sqlite_single_ver_storage_executor.cpp | 25 +++++++++---------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h index e2e150f31..dbe63aac5 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/iconnection.h @@ -22,7 +22,6 @@ #include "macro_utils.h" namespace DistributedDB { - class IConnection { public: IConnection(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index ebe347c00..31ae8b029 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -255,7 +255,7 @@ static size_t GetDataItemSerialSize(const DataItem &item, size_t appendLen) } static bool CanHoldDeletedData(const std::vector &dataItems, const DataSizeSpecInfo &dataSizeInfo, - size_t appendLen) + size_t appendLen) { bool reachThreshold = (dataItems.size() >= dataSizeInfo.packetSize); for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index cc14c2ed0..52e65653b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -664,20 +664,19 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataWithQuery(sqlite3_stmt *fullStmt, LOGE("Get next changed data failed. %d", errCode); return errCode; } - if (!isMatchItemFinished && matchItem.key == fullItem.key) { - errCode = AppendDataItem(dataItems, matchItem, dataTotalSize, appendLength, dataSizeInfo); - if (errCode == -E_UNFINISHED) { - goto END; - } - break; // step to next match data - } else { - DBCommon::CalcValueHash(fullItem.key, fullItem.key); + bool matchData = true; + if (isMatchItemFinished || matchItem.key != fullItem.key) { + matchData = false; // got miss query data + DBCommon::CalcValueHash(fullItem.key, fullItem.key); // set and send key with hash_key Value().swap(fullItem.value); // not send value when data miss query - fullItem.flag |= DataItem::REMOTE_DEVICE_DATA_MISS_QUERY; - errCode = AppendDataItem(dataItems, fullItem, dataTotalSize, appendLength, dataSizeInfo); - if (errCode == -E_UNFINISHED) { - goto END; - } + fullItem.flag |= DataItem::REMOTE_DEVICE_DATA_MISS_QUERY; // mark with miss query flag + } + errCode = AppendDataItem(dataItems, fullItem, dataTotalSize, appendLength, dataSizeInfo); + if (errCode == -E_UNFINISHED) { + goto END; + } + if (matchData) { + break; // step to next match data } } } -- Gitee From 6bb864e84f7ccc92d859a4713ac10050b6093c97 Mon Sep 17 00:00:00 2001 From: zwtmichael Date: Thu, 21 Apr 2022 17:01:57 +0800 Subject: [PATCH 14/22] fix review issue Signed-off-by: zwtmichael --- .../libs/distributeddb/common/src/user_change_monitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/user_change_monitor.cpp b/services/distributeddataservice/libs/distributeddb/common/src/user_change_monitor.cpp index 6add9c4d4..469933dab 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/user_change_monitor.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/user_change_monitor.cpp @@ -49,7 +49,7 @@ void UserChangeMonitor::Stop() if (!isStarted_) { return; } - if (userNotifier_ == nullptr) { + if (userNotifier_ != nullptr) { userNotifier_->UnRegisterEventType(USER_ACTIVE_EVENT); userNotifier_->UnRegisterEventType(USER_NON_ACTIVE_EVENT); userNotifier_->UnRegisterEventType(USER_ACTIVE_TO_NON_ACTIVE_EVENT); -- Gitee From efbe0fdaa096b7ab6c5597ab40ed50662864410a Mon Sep 17 00:00:00 2001 From: zwtmichael Date: Sun, 24 Apr 2022 15:48:41 +0800 Subject: [PATCH 15/22] fix review issue Signed-off-by: zwtmichael --- .../common/include/auto_launch.h | 2 +- .../distributeddb/common/src/auto_launch.cpp | 2 +- .../common/src/user_change_monitor.cpp | 2 +- .../communicator/include/icommunicator.h | 18 +++++++++++++-- .../communicator/src/communicator.cpp | 4 ++-- .../communicator/src/communicator.h | 4 ++-- .../distributeddb/syncer/src/ability_sync.cpp | 16 ++------------ .../distributeddb/syncer/src/ability_sync.h | 2 -- .../syncer/src/communicator_proxy.cpp | 4 ++-- .../syncer/src/communicator_proxy.h | 4 ++-- .../syncer/src/single_ver_data_sync.cpp | 15 +------------ .../syncer/src/single_ver_data_sync.h | 2 -- .../syncer/src/single_ver_data_sync_utils.cpp | 22 +++++++++++-------- .../distributeddb/syncer/src/time_helper.cpp | 9 +------- .../common/syncer/mock_communicator.h | 4 ++-- .../common/syncer/virtual_communicator.cpp | 6 ++--- .../common/syncer/virtual_communicator.h | 4 ++-- .../syncer/virtual_time_sync_communicator.cpp | 5 +++-- .../syncer/virtual_time_sync_communicator.h | 4 ++-- 19 files changed, 56 insertions(+), 73 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h index fdb79da5f..605d87435 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h @@ -133,7 +133,7 @@ protected: void UpdateGlobalMap(std::map> &doOpenMap); - void ReceiveUnknownIdentifierCallBackTask(const std::string &identifier, const std::string userId); + void ReceiveUnknownIdentifierCallBackTask(const std::string &identifier, const std::string &userId); void ConnectionLifeCycleCallback(const std::string &identifier, const std::string &userId); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp index 717e3c8a3..f5ec0ff2d 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp @@ -675,7 +675,7 @@ void AutoLaunch::UpdateGlobalMap(std::mapIsDeviceOnline(device); } -int Communicator::SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config) +int Communicator::SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config) { return SendMessage(dstTarget, inMsg, config, nullptr); } -int Communicator::SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config, +int Communicator::SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config, const OnSendEnd &onEnd) { if (dstTarget.empty() || inMsg == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h index 17c8604e2..c1f2ce39d 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h @@ -51,8 +51,8 @@ public: // Get the protocol version of remote target. Return -E_NOT_FOUND if no record. int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const override; - int SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config) override; - int SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config, + int SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config) override; + int SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config, const OnSendEnd &onEnd) override; // Call by CommunicatorAggregator directly diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp index 71000def9..94d751051 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp @@ -377,7 +377,7 @@ int AbilitySync::SyncStart(uint32_t sessionId, uint32_t sequenceId, uint16_t rem message->SetSessionId(sessionId); message->SetSequenceId(sequenceId); SendConfig conf; - SetSendConfig(deviceId_, false, SEND_TIME_OUT, conf); + SetSendConfigParam(storageInterface_->GetDbProperties(), deviceId_, false, SEND_TIME_OUT, conf); errCode = communicator_->SendMessage(deviceId_, message, conf, handler); if (errCode != E_OK) { LOGE("[AbilitySync][SyncStart] SendPacket failed, err %d", errCode); @@ -1112,7 +1112,7 @@ int AbilitySync::SendAck(const Message *inMsg, const AbilitySyncAckPacket &ackPa ackMessage->SetSessionId(inMsg->GetSessionId()); ackMessage->SetSequenceId(inMsg->GetSequenceId()); SendConfig conf; - SetSendConfig(deviceId_, false, SEND_TIME_OUT, conf); + SetSendConfigParam(storageInterface_->GetDbProperties(), deviceId_, false, SEND_TIME_OUT, conf); errCode = communicator_->SendMessage(deviceId_, ackMessage, conf); if (errCode != E_OK) { LOGE("[AbilitySync][SendAck] SendPacket failed, err %d", errCode); @@ -1174,18 +1174,6 @@ int AbilitySync::HandleRelationAckSchemaParam(const AbilitySyncAckPacket *recvPa return errCode; } -void AbilitySync::SetSendConfig(const std::string &dstTarget, bool nonBlock, uint32_t timeout, SendConfig &sendConf) -{ - sendConf.nonBlock = nonBlock; - sendConf.timeout = timeout; - sendConf.isNeedExtendHead = storageInterface_->GetDbProperties().GetBoolProp(KvDBProperties::SYNC_DUAL_TUPLE_MODE, - false); - sendConf.paramInfo.appId = storageInterface_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); - sendConf.paramInfo.userId = storageInterface_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); - sendConf.paramInfo.storeId = storageInterface_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); - sendConf.paramInfo.dstTarget = dstTarget; -} - int AbilitySync::AckRecvWithHighVersion(const Message *message, ISyncTaskContext *context, const AbilitySyncAckPacket *packet) { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h index 3605e6173..05f0465ff 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h @@ -235,8 +235,6 @@ private: RelationalSyncOpinion MakeRelationSyncOpnion(const AbilitySyncRequestPacket *packet, const std::string &remoteSchema) const; - void SetSendConfig(const std::string &dstTarget, bool nonBlock, uint32_t timeout, SendConfig &sendConf); - int AckRecvWithHighVersion(const Message *message, ISyncTaskContext *context, const AbilitySyncAckPacket *packet); ICommunicator *communicator_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp index 3eef85f24..7dae8b409 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp @@ -193,12 +193,12 @@ int CommunicatorProxy::GetRemoteCommunicatorVersion(const std::string &target, u return -E_NOT_INIT; } -int CommunicatorProxy::SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config) +int CommunicatorProxy::SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config) { return SendMessage(dstTarget, inMsg, config, nullptr); } -int CommunicatorProxy::SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config, +int CommunicatorProxy::SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config, const OnSendEnd &onEnd) { ICommunicator *targetCommunicator = nullptr; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h index 055bffcb5..08b0c6ad9 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h @@ -42,8 +42,8 @@ public: bool IsDeviceOnline(const std::string &device) const override; int GetLocalIdentity(std::string &outTarget) const override; int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const override; - int SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config) override; - int SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config, + int SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config) override; + int SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config, const OnSendEnd &onEnd) override; // Set an Main communicator for this database, used userid & appId & storeId diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index e7c4269db..8796f5144 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -237,7 +237,7 @@ int SingleVerDataSync::Send(SingleVerSyncTaskContext *context, const Message *me startFeedDogRet = context->StartFeedDogForSync(time, SyncDirectionFlag::SEND); } SendConfig sendConfig; - SetSendConfig(context->GetDeviceId(), false, SEND_TIME_OUT, sendConfig); + SetSendConfigParam(storage_->GetDbProperties(), context->GetDeviceId(), false, SEND_TIME_OUT, sendConfig); int errCode = communicateHandle_->SendMessage(context->GetDeviceId(), message, sendConfig, handler); if (errCode != E_OK) { LOGE("[DataSync][Send] send message failed, errCode=%d", errCode); @@ -2028,17 +2028,4 @@ void SingleVerDataSync::ClearDataMsg() { msgSchedule_.ClearMsg(); } - -void SingleVerDataSync::SetSendConfig(const std::string &dstTarget, bool nonBlock, uint32_t timeout, - SendConfig &sendConf) -{ - sendConf.nonBlock = nonBlock; - sendConf.timeout = timeout; - sendConf.isNeedExtendHead = storage_->GetDbProperties().GetBoolProp(KvDBProperties::SYNC_DUAL_TUPLE_MODE, - false); - sendConf.paramInfo.appId = storage_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); - sendConf.paramInfo.userId = storage_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); - sendConf.paramInfo.storeId = storage_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); - sendConf.paramInfo.dstTarget = dstTarget; -} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h index 619696e8c..610537fa9 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h @@ -243,8 +243,6 @@ protected: int SendControlAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, uint32_t controlCmdType, const CommErrHandler &handler = nullptr); - void SetSendConfig(const std::string &dstTarget, bool nonBlock, uint32_t timeout, SendConfig &sendConf); - uint32_t mtuSize_; SyncGenericInterface* storage_; ICommunicator* communicateHandle_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_utils.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_utils.cpp index 9f96f97a7..a2bc1d620 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_utils.cpp @@ -163,15 +163,19 @@ int SingleVerDataSyncUtils::RunPermissionCheck(SingleVerSyncTaskContext *context std::string userId = storage->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); std::string storeId = storage->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); uint8_t flag; - if (mode == SyncModeType::PUSH) { - flag = CHECK_FLAG_RECEIVE; - } else if (mode == SyncModeType::PULL) { - flag = CHECK_FLAG_SEND; - } else if (mode == SyncModeType::PUSH_AND_PULL) { - flag = CHECK_FLAG_SEND | CHECK_FLAG_RECEIVE; - } else { - // before add permissionCheck, PushStart packet and pullResponse packet do not setMode. - flag = CHECK_FLAG_RECEIVE; + switch (mode) { + case SyncModeType::PUSH: + flag = CHECK_FLAG_RECEIVE; + break; + case SyncModeType::PULL: + flag = CHECK_FLAG_SEND; + break; + case SyncModeType::PUSH_AND_PULL: + flag = CHECK_FLAG_SEND | CHECK_FLAG_RECEIVE; + break; + default: + flag = CHECK_FLAG_RECEIVE; + break; } int errCode = E_OK; if (storage->GetInterfaceType() != ISyncInterface::SYNC_RELATION) { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.cpp index c54b7fa6d..0ea1f4f22 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.cpp @@ -118,13 +118,6 @@ int TimeHelper::SaveLocalTimeOffset(TimeOffset offset) void TimeHelper::SetSendConfig(const std::string &dstTarget, bool nonBlock, uint32_t timeout, SendConfig &sendConf) { - sendConf.nonBlock = nonBlock; - sendConf.timeout = timeout; - sendConf.isNeedExtendHead = storage_->GetDbProperties().GetBoolProp(KvDBProperties::SYNC_DUAL_TUPLE_MODE, - false); - sendConf.paramInfo.appId = storage_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); - sendConf.paramInfo.userId = storage_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); - sendConf.paramInfo.storeId = storage_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); - sendConf.paramInfo.dstTarget = dstTarget; + SetSendConfigParam(storage_->GetDbProperties(), dstTarget, false, SEND_TIME_OUT, sendConf); } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_communicator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_communicator.h index b927f3216..aab37dff6 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_communicator.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_communicator.h @@ -24,8 +24,8 @@ class MockCommunicator : public ICommunicator { public: MOCK_CONST_METHOD1(GetLocalIdentity, int(std::string &)); MOCK_CONST_METHOD2(GetRemoteCommunicatorVersion, int(const std::string &, uint16_t &)); - MOCK_METHOD3(SendMessage, int(const std::string &, const Message *, SendConfig &)); - MOCK_METHOD4(SendMessage, int(const std::string &, const Message *, SendConfig &, const OnSendEnd &)); + MOCK_METHOD3(SendMessage, int(const std::string &, const Message *, const SendConfig &)); + MOCK_METHOD4(SendMessage, int(const std::string &, const Message *, const SendConfig &, const OnSendEnd &)); MOCK_CONST_METHOD0(GetCommunicatorMtuSize, uint32_t(void)); MOCK_CONST_METHOD1(GetCommunicatorMtuSize, uint32_t(const std::string &)); MOCK_CONST_METHOD0(GetTimeout, uint32_t(void)); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp index c8c946e7d..caed2ea27 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp @@ -43,13 +43,13 @@ void VirtualCommunicator::Activate() { } -int VirtualCommunicator::SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config) +int VirtualCommunicator::SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config) { return SendMessage(dstTarget, inMsg, config, nullptr); } -int VirtualCommunicator::SendMessage(const std::string &dstTarget, const Message *inMsg, - SendConfig &config, const OnSendEnd &onEnd) +int VirtualCommunicator::SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config, + const OnSendEnd &onEnd) { AutoLock lock(this); if (IsKilled()) { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h index ec7f766d1..bd68e82f5 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h @@ -51,8 +51,8 @@ public: uint32_t GetTimeout(const std::string &target) const override; int GetLocalIdentity(std::string &outTarget) const override; - int SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config) override; - int SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config, + int SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config) override; + int SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config, const OnSendEnd &onEnd) override; int GetRemoteCommunicatorVersion(const std::string &deviceId, uint16_t &version) const override; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp index fa20e0c2c..5c42b4261 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp @@ -82,13 +82,14 @@ int VirtualTimeSyncCommunicator::GetLocalIdentity(std::string &outTarget) const return 0; } -int VirtualTimeSyncCommunicator::SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config) +int VirtualTimeSyncCommunicator::SendMessage(const std::string &dstTarget, const Message *inMsg, + const SendConfig &config) { return SendMessage(dstTarget, inMsg, config, nullptr); } int VirtualTimeSyncCommunicator::SendMessage(const std::string &dstTarget, const Message *inMsg, - SendConfig &config, const OnSendEnd &onEnd) + const SendConfig &config, const OnSendEnd &onEnd) { if (!isEnable_) { LOGD("[VirtualTimeSyncCommunicator]the VirtualTimeSyncCommunicator disabled!"); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h index 410f587d4..e3165a464 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h @@ -57,8 +57,8 @@ public: // Get local target name for identify self int GetLocalIdentity(std::string &outTarget) const override; - int SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config) override; - int SendMessage(const std::string &dstTarget, const Message *inMsg, SendConfig &config, + int SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config) override; + int SendMessage(const std::string &dstTarget, const Message *inMsg, const SendConfig &config, const OnSendEnd &onEnd) override; int GetRemoteCommunicatorVersion(const std::string &deviceId, uint16_t &version) const override; -- Gitee From ef5ed7862a168fb2c28d14beeaa26fca55fe215d Mon Sep 17 00:00:00 2001 From: zqq Date: Sat, 9 Apr 2022 16:27:40 +0800 Subject: [PATCH 16/22] waiting all message task exist when sync engine closing Signed-off-by: zqq --- .../distributeddb/syncer/src/sync_engine.cpp | 67 ++++++++++++------- .../distributeddb/syncer/src/sync_engine.h | 6 ++ 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp index 2466b6c0f..5b666e128 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp @@ -118,6 +118,10 @@ int SyncEngine::Close() UnRegCommunicatorsCallback(); StopAutoSubscribeTimer(); + std::unique_lock closeLock(execTaskCountLock_); + execTaskCv_.wait(closeLock, [&]() { + return execTaskCount_ == 0; + }); // Clear SyncContexts { std::unique_lock lock(contextMapLock_); @@ -355,10 +359,7 @@ void SyncEngine::RemoteDataChangedTask(ISyncTaskContext *context, const ICommuni void SyncEngine::ScheduleTaskOut(ISyncTaskContext *context, const ICommunicator *communicator) { (void)DealMsgUtilQueueEmpty(); - { - std::lock_guard lock(queueLock_); - execTaskCount_--; - } + DecExecTaskCount(); RefObject::DecObjRef(communicator); RefObject::DecObjRef(context); } @@ -377,19 +378,23 @@ int SyncEngine::DealMsgUtilQueueEmpty() queueCacheSize_ -= GetMsgSize(inMsg); } + IncExecTaskCount(); // it will deal with the first message in queue, we should increase object reference counts and sure that resources // could be prevented from destroying by other threads. - ISyncTaskContext *nextContext = GetConextForMsg(inMsg->GetTarget(), errCode); - if (errCode != E_OK) { - delete inMsg; - inMsg = nullptr; - return errCode; - } - errCode = ScheduleDealMsg(nextContext, inMsg); + do { + ISyncTaskContext *nextContext = GetConextForMsg(inMsg->GetTarget(), errCode); + if (errCode != E_OK) { + break; + } + errCode = ScheduleDealMsg(nextContext, inMsg); + if (errCode != E_OK) { + RefObject::DecObjRef(nextContext); + } + } while (false); if (errCode != E_OK) { - RefObject::DecObjRef(nextContext); delete inMsg; inMsg = nullptr; + DecExecTaskCount(); } return errCode; } @@ -425,13 +430,10 @@ int SyncEngine::ScheduleDealMsg(ISyncTaskContext *context, Message *inMsg) { if (inMsg == nullptr) { LOGE("[SyncEngine] MessageReciveCallback inMsg is null!"); + DecExecTaskCount(); return E_OK; } RefObject::IncObjRef(communicatorProxy_); - { - std::lock_guard incLock(queueLock_); - execTaskCount_++; - } int errCode = E_OK; // deal remote local data changed message if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { @@ -444,10 +446,6 @@ int SyncEngine::ScheduleDealMsg(ISyncTaskContext *context, Message *inMsg) if (errCode != E_OK) { LOGE("[SyncEngine] MessageReciveCallbackTask Schedule failed err %d", errCode); RefObject::DecObjRef(communicatorProxy_); - { - std::lock_guard decLock(queueLock_); - execTaskCount_--; - } } return errCode; } @@ -495,14 +493,20 @@ int SyncEngine::MessageReciveCallbackInner(const std::string &targetDev, Message } } + IncExecTaskCount(); int errCode = E_OK; - ISyncTaskContext *nextContext = GetConextForMsg(targetDev, errCode); + do { + ISyncTaskContext *nextContext = GetConextForMsg(targetDev, errCode); + if (errCode != E_OK) { + break; + } + LOGD("[SyncEngine] MessageReciveCallback MSG ID = %d", inMsg->GetMessageId()); + errCode = ScheduleDealMsg(nextContext, inMsg); + } while (false); if (errCode != E_OK) { - return errCode; + DecExecTaskCount(); } - - LOGD("[SyncEngine] MessageReciveCallback MSG ID = %d", inMsg->GetMessageId()); - return ScheduleDealMsg(nextContext, inMsg); + return errCode; } void SyncEngine::PutMsgIntoQueue(const std::string &targetDev, Message *inMsg, int msgSize) @@ -1022,4 +1026,17 @@ void SyncEngine::SchemaChange() context->SchemaChange(); } } + +void SyncEngine::IncExecTaskCount() +{ + std::lock_guard incLock(execTaskCountLock_); + execTaskCount_++; +} + +void SyncEngine::DecExecTaskCount() +{ + std::lock_guard incLock(execTaskCountLock_); + execTaskCount_--; + execTaskCv_.notify_all(); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h index f942302f3..9cb3f6fd2 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h @@ -189,6 +189,10 @@ private: static uint8_t GetPermissionCheckFlag(bool isAutoSync, int syncMode); + void IncExecTaskCount(); + + void DecExecTaskCount(); + ICommunicator *communicator_; DeviceManager *deviceManager_; std::function onRemoteDataChanged_; @@ -213,6 +217,8 @@ private: // key: device value: equalIdentifier std::map equalIdentifierMap_; + std::mutex execTaskCountLock_; + std::condition_variable execTaskCv_; }; } // namespace DistributedDB -- Gitee From 104cfede6e68f1c9ebd49d48d38b75c0a40693a8 Mon Sep 17 00:00:00 2001 From: zqq Date: Mon, 18 Apr 2022 09:17:30 +0800 Subject: [PATCH 17/22] avoid fake wake up Signed-off-by: zqq --- .../libs/distributeddb/syncer/src/sync_engine.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp index 5b666e128..373752ef9 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp @@ -1035,8 +1035,10 @@ void SyncEngine::IncExecTaskCount() void SyncEngine::DecExecTaskCount() { - std::lock_guard incLock(execTaskCountLock_); - execTaskCount_--; + { + std::lock_guard decLock(execTaskCountLock_); + execTaskCount_--; + } execTaskCv_.notify_all(); } } // namespace DistributedDB -- Gitee From 0015226c866b213d51c70f8dbd753fa58c94e1cc Mon Sep 17 00:00:00 2001 From: zqq Date: Thu, 21 Apr 2022 10:12:16 +0800 Subject: [PATCH 18/22] add exec count when receive msg Signed-off-by: zqq --- .../distributeddb/syncer/src/sync_engine.cpp | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp index 373752ef9..9fad25282 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp @@ -452,10 +452,12 @@ int SyncEngine::ScheduleDealMsg(ISyncTaskContext *context, Message *inMsg) void SyncEngine::MessageReciveCallback(const std::string &targetDev, Message *inMsg) { + IncExecTaskCount(); int errCode = MessageReciveCallbackInner(targetDev, inMsg); if (errCode != E_OK) { delete inMsg; inMsg = nullptr; + DecExecTaskCount(); LOGE("[SyncEngine] MessageReciveCallback failed!"); } } @@ -487,26 +489,21 @@ int SyncEngine::MessageReciveCallbackInner(const std::string &targetDev, Message return -E_BUSY; } - if (execTaskCount_ >= MAX_EXEC_NUM) { + if (execTaskCount_ > MAX_EXEC_NUM) { PutMsgIntoQueue(targetDev, inMsg, msgSize); + // task dont exec here + DecExecTaskCount(); return E_OK; } } - IncExecTaskCount(); int errCode = E_OK; - do { - ISyncTaskContext *nextContext = GetConextForMsg(targetDev, errCode); - if (errCode != E_OK) { - break; - } - LOGD("[SyncEngine] MessageReciveCallback MSG ID = %d", inMsg->GetMessageId()); - errCode = ScheduleDealMsg(nextContext, inMsg); - } while (false); + ISyncTaskContext *nextContext = GetConextForMsg(targetDev, errCode); if (errCode != E_OK) { - DecExecTaskCount(); + return errCode; } - return errCode; + LOGD("[SyncEngine] MessageReciveCallback MSG ID = %d", inMsg->GetMessageId()); + return ScheduleDealMsg(nextContext, inMsg); } void SyncEngine::PutMsgIntoQueue(const std::string &targetDev, Message *inMsg, int msgSize) -- Gitee From c7e838f70a9c77429412591411c2f44482c80151 Mon Sep 17 00:00:00 2001 From: zqq Date: Sun, 24 Apr 2022 11:16:08 +0800 Subject: [PATCH 19/22] add timeout when waiting exec task count Signed-off-by: zqq --- .../libs/distributeddb/syncer/src/sync_engine.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp index 9fad25282..52eedc190 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp @@ -119,9 +119,11 @@ int SyncEngine::Close() StopAutoSubscribeTimer(); std::unique_lock closeLock(execTaskCountLock_); - execTaskCv_.wait(closeLock, [&]() { - return execTaskCount_ == 0; - }); + bool isTimeout = execTaskCv_.wait_for(closeLock, std::chrono::milliseconds(DBConstant::MIN_TIMEOUT), + [this]() { return execTaskCount_ == 0; }); + if (!isTimeout) { + LOGD("SyncEngine Close with executing task!"); + } // Clear SyncContexts { std::unique_lock lock(contextMapLock_); @@ -366,6 +368,9 @@ void SyncEngine::ScheduleTaskOut(ISyncTaskContext *context, const ICommunicator int SyncEngine::DealMsgUtilQueueEmpty() { + if (!isActive_) { + return -E_BUSY; // db is closing just return + } int errCode = E_OK; Message *inMsg = nullptr; { -- Gitee From bbd905a275daf4637deea653a829d2b40d07253a Mon Sep 17 00:00:00 2001 From: zqq Date: Tue, 19 Apr 2022 15:51:57 +0800 Subject: [PATCH 20/22] TimeTickMonitor async notify TIME_CHANGE_EVENT Signed-off-by: zqq --- .../libs/distributeddb/common/src/time_tick_monitor.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.cpp b/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.cpp index d43fadc18..1bb415e68 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.cpp @@ -123,7 +123,13 @@ int TimeTickMonitor::TimeTick(TimerId timerId) int64_t changedOffset = systemOffset - monotonicOffset; if (std::abs(changedOffset) > MAX_NOISE) { LOGI("Local system time may be changed! changedOffset %ld", changedOffset); - timeChangedNotifier_->NotifyEvent(TIME_CHANGE_EVENT, &changedOffset); + int ret = RuntimeContext::GetInstance()->ScheduleTask([this, changedOffset](){ + int64_t offset = changedOffset; + timeChangedNotifier_->NotifyEvent(TIME_CHANGE_EVENT, &offset); + }); + if (ret != E_OK) { + LOGE("TimeTickMonitor ScheduleTask failed %d", ret); + } } return E_OK; } -- Gitee From 4ad434b0c6bab090974994819aa0fb917026796e Mon Sep 17 00:00:00 2001 From: zwtmichael Date: Mon, 25 Apr 2022 10:35:51 +0800 Subject: [PATCH 21/22] fix review issue Signed-off-by: zwtmichael --- .../common/include/auto_launch.h | 2 +- .../distributeddb/common/src/auto_launch.cpp | 4 +- .../common/src/user_change_monitor.cpp | 43 +++++++++---------- .../storage/src/relational_store_instance.cpp | 4 ++ 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h index 605d87435..a6da51344 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h @@ -143,7 +143,7 @@ protected: int AutoLaunchExt(const std::string &identifier, const std::string &userId); - void AutoLaunchExtTask(const std::string identifier, const std::string userId, AutoLaunchItem autoLaunchItem); + void AutoLaunchExtTask(const std::string &identifier, const std::string &userId, AutoLaunchItem &autoLaunchItem); void ExtObserverFunc(const KvDBCommitNotifyData ¬ifyData, const std::string &identifier, const std::string &userId); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp index f5ec0ff2d..f8976ce52 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp @@ -812,8 +812,8 @@ int AutoLaunch::AutoLaunchExt(const std::string &identifier, const std::string & return errCode; } -void AutoLaunch::AutoLaunchExtTask(const std::string identifier, const std::string userId, - AutoLaunchItem autoLaunchItem) +void AutoLaunch::AutoLaunchExtTask(const std::string &identifier, const std::string &userId, + AutoLaunchItem &autoLaunchItem) { { std::lock_guard autoLock(extLock_); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/user_change_monitor.cpp b/services/distributeddataservice/libs/distributeddb/common/src/user_change_monitor.cpp index a0bff012f..f670ce318 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/user_change_monitor.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/user_change_monitor.cpp @@ -82,30 +82,29 @@ int UserChangeMonitor::PrepareNotifierChain() if (userNotifier_ != nullptr) { return E_OK; } + userNotifier_ = new (std::nothrow) NotificationChain(); if (userNotifier_ == nullptr) { - userNotifier_ = new (std::nothrow) NotificationChain(); - if (userNotifier_ == nullptr) { - return -E_OUT_OF_MEMORY; - } - errCode = userNotifier_->RegisterEventType(USER_ACTIVE_EVENT); - if (errCode != E_OK) { - RefObject::KillAndDecObjRef(userNotifier_); - userNotifier_ = nullptr; - return errCode; - } - errCode = userNotifier_->RegisterEventType(USER_NON_ACTIVE_EVENT); - if (errCode != E_OK) { - RefObject::KillAndDecObjRef(userNotifier_); - userNotifier_ = nullptr; - return errCode; - } - errCode = userNotifier_->RegisterEventType(USER_ACTIVE_TO_NON_ACTIVE_EVENT); - if (errCode != E_OK) { - RefObject::KillAndDecObjRef(userNotifier_); - userNotifier_ = nullptr; - return errCode; - } + return -E_OUT_OF_MEMORY; } + errCode = userNotifier_->RegisterEventType(USER_ACTIVE_EVENT); + if (errCode != E_OK) { + goto ERROR_HANDLE; + } + errCode = userNotifier_->RegisterEventType(USER_NON_ACTIVE_EVENT); + if (errCode != E_OK) { + userNotifier_->UnRegisterEventType(USER_ACTIVE_EVENT); + goto ERROR_HANDLE; + } + errCode = userNotifier_->RegisterEventType(USER_ACTIVE_TO_NON_ACTIVE_EVENT); + if (errCode != E_OK) { + userNotifier_->UnRegisterEventType(USER_ACTIVE_EVENT); + userNotifier_->UnRegisterEventType(USER_NON_ACTIVE_EVENT); + goto ERROR_HANDLE; + } + return errCode; +ERROR_HANDLE: + RefObject::KillAndDecObjRef(userNotifier_); + userNotifier_ = nullptr; return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp index 443737900..fdb04ccff 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp @@ -167,6 +167,10 @@ RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); LOGD("Begin to get [%s] database connection.", STR_MASK(DBCommon::TransferStringToHex(identifier))); RelationalStoreInstance *manager = RelationalStoreInstance::GetInstance(); + if (manager == nullptr) { + errCode = -E_OUT_OF_MEMORY; + return nullptr; + } manager->EnterDBOpenCloseProcess(properties.GetStringProp(DBProperties::IDENTIFIER_DATA, "")); RelationalStoreConnection *connection = nullptr; std::string canonicalDir; -- Gitee From f9c87b2fb1de55cf515e59dfcc2d20a26b1150a6 Mon Sep 17 00:00:00 2001 From: zqq Date: Mon, 25 Apr 2022 11:54:26 +0800 Subject: [PATCH 22/22] change sesseionId from uint64 to uint32, for align problem Signed-off-by: zqq --- .../libs/distributeddb/common/include/runtime_context.h | 2 +- .../distributeddb/common/src/runtime_context_impl.cpp | 8 ++++++-- .../libs/distributeddb/common/src/runtime_context_impl.h | 4 ++-- .../libs/distributeddb/storage/src/iconnection.cpp | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h index 01f79c369..cd09bdd16 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h @@ -125,7 +125,7 @@ public: virtual int NotifyUserChanged() const = 0; // Generate global sessionId in current process - virtual uint64_t GenerateSessionId() = 0; + virtual uint32_t GenerateSessionId() = 0; protected: RuntimeContext() = default; virtual ~RuntimeContext() {} diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp index 9c02425c1..9414a1404 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp @@ -656,8 +656,12 @@ int RuntimeContextImpl::NotifyUserChanged() const return E_OK; } -uint64_t RuntimeContextImpl::GenerateSessionId() +uint32_t RuntimeContextImpl::GenerateSessionId() { - return currentSessionId_.fetch_add(1, std::memory_order_seq_cst); + uint32_t sessionId = currentSessionId_++; + if (sessionId == 0) { + sessionId = currentSessionId_++; + } + return sessionId; } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h index b998b9da5..cf7fb4c6f 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h @@ -114,7 +114,7 @@ public: // Notify TIME_CHANGE_EVENT. int NotifyUserChanged() const override; - uint64_t GenerateSessionId() override; + uint32_t GenerateSessionId() override; private: static constexpr int MAX_TP_THREADS = 10; // max threads of the task pool. static constexpr int MIN_TP_THREADS = 1; // min threads of the task pool. @@ -170,7 +170,7 @@ private: mutable std::mutex userChangeMonitorLock_; std::unique_ptr userChangeMonitor_; - std::atomic currentSessionId_; + std::atomic currentSessionId_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp index 478397e2a..620a9a31a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/iconnection.cpp @@ -30,7 +30,7 @@ uint64_t IConnection::GetConnectionId() std::lock_guard autoLock(connectionIdLock_); // check again here, may be generated after get lock if (connectionId_ == 0) { - connectionId_ = RuntimeContext::GetInstance()->GenerateSessionId(); + connectionId_ = static_cast(RuntimeContext::GetInstance()->GenerateSessionId()); } return connectionId_; } -- Gitee