From 323202ab75d658bbd68e5b5c566de922d317b6ae Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 21 Dec 2021 22:48:27 +0800 Subject: [PATCH 01/53] Add relational DB properties Signed-off-by: lianhuix --- .../relational_store_delegate_impl.h | 2 + .../relational/relational_store_instance.h | 10 ++--- .../relational/relational_store_manager.cpp | 18 ++++---- .../storage/include/kvdb_properties.h | 42 ++++++++++++++----- .../include/relational_db_sync_interface.h | 2 + .../storage/src/irelational_store.h | 2 +- .../storage/src/kvdb_properties.cpp | 38 +++++++++++++---- .../storage/src/relational_store_instance.cpp | 18 ++++---- .../relational/sqlite_relational_store.cpp | 7 ++-- .../relational/sqlite_relational_store.h | 6 +-- .../sqlite_relational_store_connection.cpp | 1 + .../sqlite_relational_store_connection.h | 2 +- ...qlite_single_relational_storage_engine.cpp | 10 ++++- ...ingle_ver_relational_storage_executor.cpp} | 10 +++++ 14 files changed, 117 insertions(+), 51 deletions(-) rename services/distributeddataservice/libs/distributeddb/storage/src/sqlite/{sqlite_single_ver_relation_storage_executor.cpp => sqlite_single_ver_relational_storage_executor.cpp} (99%) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 6dc5404c2..25d51d440 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -33,12 +33,14 @@ public: DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) override; + DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, const Query &query, bool wait) override; DBStatus RemoveDeviceData(const std::string &device) override; DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) override; + DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) override; // For connection diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h index 34316ef82..1bed16fe0 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h @@ -28,19 +28,19 @@ public: RelationalStoreInstance(); ~RelationalStoreInstance() = default; - static RelationalStoreConnection *GetDatabaseConnection(const DBProperties &properties, int &errCode); + static RelationalStoreConnection *GetDatabaseConnection(const RelationalDBProperties &properties, int &errCode); static RelationalStoreInstance *GetInstance(); int CheckDatabaseFileStatus(const std::string &id); // public for test mock - static IRelationalStore *GetDataBase(const DBProperties &properties, int &errCode); + static IRelationalStore *GetDataBase(const RelationalDBProperties &properties, int &errCode); private: - IRelationalStore *OpenDatabase(const DBProperties &properties, int &errCode); + IRelationalStore *OpenDatabase(const RelationalDBProperties &properties, int &errCode); - void RemoveKvDBFromCache(const DBProperties &properties); - void SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties); + void RemoveKvDBFromCache(const RelationalDBProperties &properties); + void SaveKvDBToCache(IRelationalStore *store, const RelationalDBProperties &properties); static RelationalStoreInstance *instance_; static std::mutex instanceLock_; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 9a4a60324..47bd5eb62 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -33,22 +33,22 @@ RelationalStoreManager::RelationalStoreManager(const std::string &appId, const s {} static void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, - const std::string &appId, const std::string &userId, DBProperties &properties) + const std::string &appId, const std::string &userId, RelationalDBProperties &properties) { - properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); - properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); - properties.SetStringProp(KvDBProperties::APP_ID, appId); - properties.SetStringProp(KvDBProperties::USER_ID, userId); - properties.SetStringProp(KvDBProperties::STORE_ID, storePath); // same as dir + properties.SetBoolProp(RelationalDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); + properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); + properties.SetStringProp(RelationalDBProperties::APP_ID, appId); + properties.SetStringProp(RelationalDBProperties::USER_ID, userId); + properties.SetStringProp(RelationalDBProperties::STORE_ID, storePath); // same as dir std::string identifier = userId + "-" + appId + "-" + storePath; std::string hashIdentifier = DBCommon::TransferHashString(identifier); - properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); + properties.SetStringProp(RelationalDBProperties::IDENTIFIER_DATA, hashIdentifier); } static const int GET_CONNECT_RETRY = 3; static const int RETRY_GET_CONN_INTER = 30; -static RelationalStoreConnection *GetOneConnectionWithRetry(const DBProperties &properties, int &errCode) +static RelationalStoreConnection *GetOneConnectionWithRetry(const RelationalDBProperties &properties, int &errCode) { for (int i = 0; i < GET_CONNECT_RETRY; i++) { auto conn = RelationalStoreInstance::GetDatabaseConnection(properties, errCode); @@ -77,7 +77,7 @@ void RelationalStoreManager::OpenStore(const std::string &path, const Relational return; } - DBProperties properties; + RelationalDBProperties properties; InitStoreProp(option, path, appId_, userId_, properties); int errCode = E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h index 6337094af..7250af7a5 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h @@ -21,12 +21,13 @@ #include #include "schema_object.h" +#include "relational_schema_object.h" namespace DistributedDB { class DBProperties { public: DBProperties() = default; - ~DBProperties() = default; + virtual ~DBProperties() = default; // Get the string property according the name std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; @@ -45,6 +46,16 @@ public: // Set the integer property for the name void SetIntProp(const std::string &name, int value); + + static const std::string CREATE_IF_NECESSARY; + static const std::string DATABASE_TYPE; + static const std::string DATA_DIR; + static const std::string USER_ID; + static const std::string APP_ID; + static const std::string STORE_ID; + static const std::string IDENTIFIER_DATA; + static const std::string IDENTIFIER_DIR; + protected: std::map stringProperties_; std::map boolProperties_; @@ -54,7 +65,7 @@ protected: class KvDBProperties final : public DBProperties { public: KvDBProperties(); - ~KvDBProperties(); + ~KvDBProperties() override; // Get the sub directory for different type database. static std::string GetStoreSubDirectory(int type); @@ -83,18 +94,10 @@ public: // The upper code will not change the schema if it is already set const SchemaObject &GetSchemaConstRef() const; - static const std::string CREATE_IF_NECESSARY; - static const std::string DATABASE_TYPE; - static const std::string DATA_DIR; - static const std::string USER_ID; - static const std::string APP_ID; - static const std::string STORE_ID; static const std::string FILE_NAME; static const std::string SYNC_MODE; static const std::string MEMORY_MODE; static const std::string ENCRYPTED_MODE; - static const std::string IDENTIFIER_DATA; - static const std::string IDENTIFIER_DIR; static const std::string FIRST_OPEN_IS_READ_ONLY; static const std::string CREATE_DIR_BY_STORE_ID_ONLY; static const std::string SECURITY_LABEL; @@ -114,6 +117,25 @@ private: CipherPassword password_; SchemaObject schema_; }; + +// TODO: move to its own file, or rename this file +class RelationalDBProperties final : public DBProperties { +public: + RelationalDBProperties(); + ~RelationalDBProperties() override; + + // is schema exist + bool IsSchemaExist() const; + + // set schema + void SetSchema(const RelationalSchemaObject &schema); + + // get schema + RelationalSchemaObject GetSchema() const; + +private: + RelationalSchemaObject schema_; +}; } // namespace DistributedDB #endif // KV_DB_PROPERTIES_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h index 94749dba3..eff6dbe54 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h @@ -38,6 +38,8 @@ public: virtual int LocalDataChanged(int notifyEvent, std::vector &queryObj) = 0; virtual int SchemaChanged(int notifyEvent) = 0; + + // TODO: create device table for each distributed table. }; } #endif // RELATIONAL_STORE diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h index 2d8cf45e0..5ff966e9e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h @@ -31,7 +31,7 @@ public: DISABLE_COPY_ASSIGN_MOVE(IRelationalStore); // Open the database. - virtual int Open(const DBProperties &kvDBProp) = 0; + virtual int Open(const RelationalDBProperties &kvDBProp) = 0; virtual void WakeUpSyncer() = 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp index ee95611ea..75e98ded8 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp @@ -18,17 +18,18 @@ #include "db_constant.h" namespace DistributedDB { -const std::string KvDBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; -const std::string KvDBProperties::DATABASE_TYPE = "databaseType"; -const std::string KvDBProperties::DATA_DIR = "dataDir"; -const std::string KvDBProperties::USER_ID = "userId"; -const std::string KvDBProperties::APP_ID = "appId"; -const std::string KvDBProperties::STORE_ID = "storeId"; +const std::string DBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; +const std::string DBProperties::DATABASE_TYPE = "databaseType"; +const std::string DBProperties::DATA_DIR = "dataDir"; +const std::string DBProperties::USER_ID = "userId"; +const std::string DBProperties::APP_ID = "appId"; +const std::string DBProperties::STORE_ID = "storeId"; +const std::string DBProperties::IDENTIFIER_DATA = "identifier"; +const std::string DBProperties::IDENTIFIER_DIR = "identifierDir"; + const std::string KvDBProperties::FILE_NAME = "fileName"; const std::string KvDBProperties::MEMORY_MODE = "memoryMode"; const std::string KvDBProperties::ENCRYPTED_MODE = "isEncryptedDb"; -const std::string KvDBProperties::IDENTIFIER_DATA = "identifier"; -const std::string KvDBProperties::IDENTIFIER_DIR = "identifierDir"; const std::string KvDBProperties::FIRST_OPEN_IS_READ_ONLY = "firstOpenIsReadOnly"; const std::string KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY = "createDirByStoreIdOnly"; const std::string KvDBProperties::SECURITY_LABEL = "securityLabel"; @@ -145,4 +146,25 @@ const SchemaObject &KvDBProperties::GetSchemaConstRef() const { return schema_; } + +RelationalDBProperties::RelationalDBProperties() +{} + +RelationalDBProperties::~RelationalDBProperties() +{} + +bool RelationalDBProperties::IsSchemaExist() const +{ + return schema_.IsSchemaValid(); +} + +void RelationalDBProperties::SetSchema(const RelationalSchemaObject &schema) +{ + schema_ = schema; +} + +RelationalSchemaObject RelationalDBProperties::GetSchema() const +{ + return schema_; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp index e48bc8230..9a304762e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp @@ -54,10 +54,10 @@ int RelationalStoreInstance::CheckDatabaseFileStatus(const std::string &id) return E_OK; } -static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCode) +static IRelationalStore *GetFromCache(const RelationalDBProperties &properties, int &errCode) { errCode = E_OK; - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); auto iter = dbs_.find(identifier); if (iter == dbs_.end()) { errCode = -E_NOT_FOUND; @@ -76,21 +76,21 @@ static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCo } // Save to IKvDB to the global map -void RelationalStoreInstance::RemoveKvDBFromCache(const DBProperties &properties) +void RelationalStoreInstance::RemoveKvDBFromCache(const RelationalDBProperties &properties) { std::lock_guard lockGuard(storeLock_); - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); dbs_.erase(identifier); } -void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties) +void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const RelationalDBProperties &properties) { if (store == nullptr) { return; } { - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); store->WakeUpSyncer(); if (dbs_.count(identifier) == 0) { dbs_.insert(std::pair(identifier, store)); @@ -98,7 +98,7 @@ void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const DBP } } -IRelationalStore *RelationalStoreInstance::OpenDatabase(const DBProperties &properties, int &errCode) +IRelationalStore *RelationalStoreInstance::OpenDatabase(const RelationalDBProperties &properties, int &errCode) { auto db = new (std::nothrow) SQLiteRelationalStore(); if (db == nullptr) { @@ -122,7 +122,7 @@ IRelationalStore *RelationalStoreInstance::OpenDatabase(const DBProperties &prop return db; } -IRelationalStore *RelationalStoreInstance::GetDataBase(const DBProperties &properties, int &errCode) +IRelationalStore *RelationalStoreInstance::GetDataBase(const RelationalDBProperties &properties, int &errCode) { std::lock_guard lockGuard(storeLock_); auto *db = GetFromCache(properties, errCode); @@ -145,7 +145,7 @@ IRelationalStore *RelationalStoreInstance::GetDataBase(const DBProperties &prope return db; } -RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const DBProperties &properties, int &errCode) +RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const RelationalDBProperties &properties, int &errCode) { IRelationalStore *db = GetDataBase(properties, errCode); if (db == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index ed7bdb058..dfaf56757 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -26,7 +26,6 @@ SQLiteRelationalStore::~SQLiteRelationalStore() delete sqliteStorageEngine_; } - // Called when a new connection created. void SQLiteRelationalStore::IncreaseConnectionCounter() { @@ -50,13 +49,13 @@ RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) return connection; } -static void InitDataBaseOption(const DBProperties &kvDBProp, OpenDbProperties &option) +static void InitDataBaseOption(const RelationalDBProperties &kvDBProp, OpenDbProperties &option) { option.uri = kvDBProp.GetStringProp(KvDBProperties::DATA_DIR, ""); option.createIfNecessary = kvDBProp.GetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, false); } -int SQLiteRelationalStore::InitStorageEngine(const DBProperties &kvDBProp) +int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &kvDBProp) { OpenDbProperties option; InitDataBaseOption(kvDBProp, option); @@ -69,7 +68,7 @@ int SQLiteRelationalStore::InitStorageEngine(const DBProperties &kvDBProp) return errCode; } -int SQLiteRelationalStore::Open(const DBProperties &properties) +int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) { sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); if (sqliteStorageEngine_ == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index f41dd0839..0a6f2ab68 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -33,7 +33,7 @@ public: ~SQLiteRelationalStore() override; RelationalStoreConnection *GetDBConnection(int &errCode) override; - int Open(const DBProperties &properties) override; + int Open(const RelationalDBProperties &properties) override; void OnClose(const std::function ¬ifier); SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode) const; @@ -43,7 +43,7 @@ public: void ReleaseDBConnection(RelationalStoreConnection *connection); - void WakeUpSyncer(); + void WakeUpSyncer() override; // for test mock const RelationalSyncAbleStorage *GetStorageEngine() @@ -61,7 +61,7 @@ private: SQLiteSingleRelationalStorageEngine *sqliteStorageEngine_ = nullptr; void IncreaseConnectionCounter(); - int InitStorageEngine(const DBProperties &kvDBProp); + int InitStorageEngine(const RelationalDBProperties &kvDBProp); std::mutex connectMutex_; std::atomic connectionCount_ = 0; std::vector> closeNotifiers_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index d322575d7..a66269b12 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -182,6 +182,7 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) syncParam.relationOnComplete = info.onComplete; syncParam.syncQuery = QuerySyncObject(info.query); syncParam.onFinalize = [this]() { DecObjRef(this); }; + // TODO: check if table permit sync or not int errCode = store->Sync(syncParam); if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 42184112f..020560850 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -42,7 +42,7 @@ public: protected: - int Pragma(int cmd, void *parameter); + int Pragma(int cmd, void *parameter) override; private: SQLiteSingleVerRelationalStorageExecutor *GetExecutor(bool isWrite, int &errCode) const; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index ab12fd5d8..a47129d7c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -57,7 +57,7 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage return errCode; } do { - errCode = Upgrade(db); + errCode = Upgrade(db); // cerate meta_data table. if (errCode != E_OK) { break; } @@ -66,6 +66,14 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage if (errCode != E_OK) { break; } + + // TODO: Get and parse relational schema from meta table + + // TODO: save log table version into meta data + + // TODO: clean the device table + + handle = NewSQLiteStorageExecutor(db, isWrite, false); if (handle == nullptr) { LOGE("[Relational] New SQLiteStorageExecutor[%d] for the pool failed.", isWrite); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp similarity index 99% rename from services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp rename to services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 167adfdb2..be162f3b0 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -36,6 +36,16 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std:: return errCode; } + // TODO: check if compatible upgrade or not + + // TODO: add not permit sync flag if not compatible upgrade + + // TODO: add analysed table to schema. + + // TODO: reset permit sync flag + + // TODO: upgrade device table + // create log table errCode = SQLiteUtils::CreateRelationalLogTable(dbHandle_, tableName); if (errCode != E_OK) { -- Gitee From 04db30d21358d0bbe63ec1f7cf3ed176b1783572 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 22 Dec 2021 11:45:51 +0800 Subject: [PATCH 02/53] Add relationa build script Signed-off-by: lianhuix --- .../libs/distributeddb/BUILD.gn | 21 +++++++++++++++++++ .../libs/distributeddb/test/BUILD.gn | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index c763dcedc..05ae98eac 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -18,13 +18,16 @@ config("distrdb_config") { "include", "interfaces/include", "interfaces/src", + "interfaces/src/relational", "common/include", + "common/include/relational", "communicator/include", "storage/include", "storage/src", "storage/src/multiver", "storage/src/operation", "storage/src/sqlite", + "storage/src/sqlite/relational", "storage/src/upgrader", "syncer/include", "syncer/src", @@ -41,6 +44,7 @@ config("distrdb_config") { "USING_DB_JSON_EXTRACT_AUTOMATICALLY", "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", + "RELATIONAL_STORE", ] } @@ -48,6 +52,7 @@ config("distrdb_public_config") { visibility = [ "*:*" ] include_dirs = [ "interfaces/include", + "interfaces/include/relational", "include", ] } @@ -82,6 +87,7 @@ ohos_shared_library("distributeddb") { "common/src/query.cpp", "common/src/query_expression.cpp", "common/src/ref_object.cpp", + "common/src/relational/relational_schema_object.cpp", "common/src/runtime_context.cpp", "common/src/runtime_context_impl.cpp", "common/src/schema_object.cpp", @@ -115,6 +121,12 @@ ohos_shared_library("distributeddb") { "interfaces/src/kv_store_nb_delegate_impl.cpp", "interfaces/src/kv_store_result_set_impl.cpp", "interfaces/src/kv_store_snapshot_delegate_impl.cpp", + "interfaces/src/relational/data_value.cpp", + "interfaces/src/relational/relational_store_delegate_impl.cpp", + "interfaces/src/relational/relational_store_manager.cpp", + "interfaces/src/relational/relational_store_sqlite_ext.cpp", + "interfaces/src/relational/runtime_config.cpp", + "storage/src/data_transformer.cpp", "storage/src/default_factory.cpp", "storage/src/generic_kvdb.cpp", "storage/src/generic_kvdb_connection.cpp", @@ -145,10 +157,16 @@ ohos_shared_library("distributeddb") { "storage/src/operation/multi_ver_database_oper.cpp", "storage/src/operation/single_ver_database_oper.cpp", "storage/src/package_file.cpp", + "storage/src/relational_store_connection.cpp", + "storage/src/relational_store_instance.cpp", + "storage/src/relational_sync_able_storage.cpp", "storage/src/result_entries_window.cpp", "storage/src/single_ver_natural_store_commit_notify_data.cpp", "storage/src/sqlite/query_object.cpp", "storage/src/sqlite/query_sync_object.cpp", + "storage/src/sqlite/relational/sqlite_relational_store.cpp", + "storage/src/sqlite/relational/sqlite_relational_store_connection.cpp", + "storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp", "storage/src/sqlite/sqlite_local_kvdb.cpp", "storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -162,6 +180,7 @@ ohos_shared_library("distributeddb") { "storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "storage/src/sqlite/sqlite_single_ver_result_set.cpp", "storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", "storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", @@ -174,6 +193,7 @@ ohos_shared_library("distributeddb") { "storage/src/storage_engine.cpp", "storage/src/storage_engine_manager.cpp", "storage/src/storage_executor.cpp", + "storage/src/sync_able_engine.cpp", "storage/src/sync_able_kvdb.cpp", "storage/src/sync_able_kvdb_connection.cpp", "storage/src/upgrader/single_ver_database_upgrader.cpp", @@ -194,6 +214,7 @@ ohos_shared_library("distributeddb") { "syncer/src/single_ver_data_sync.cpp", "syncer/src/single_ver_data_sync_with_sliding_window.cpp", "syncer/src/single_ver_kv_syncer.cpp", + "syncer/src/single_ver_relational_syncer.cpp", "syncer/src/single_ver_serialize_manager.cpp", "syncer/src/single_ver_sync_engine.cpp", "syncer/src/single_ver_sync_state_machine.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 7b98fee3f..c181d617b 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -25,14 +25,18 @@ config("module_private_config") { "./unittest/common/interfaces", "../include", "../interfaces/include", + "../interfaces/include/relational", "../interfaces/src", + "../interfaces/src/relational", "../storage/include", "../storage/src", "../storage/src/multiver", "../storage/src/operation", "../storage/src/sqlite", + "../storage/src/sqlite/relational", "../storage/src/upgrader", "../common/include", + "../common/include/relational", "../common/src", "../communicator/include", "../communicator/src", @@ -53,6 +57,7 @@ config("module_private_config") { "LOW_LEVEL_MEM_DEV", "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", + "RELATIONAL_STORE", ] } @@ -85,6 +90,7 @@ ohos_source_set("src_file") { "../common/src/query.cpp", "../common/src/query_expression.cpp", "../common/src/ref_object.cpp", + "../common/src/relational/relational_schema_object.cpp", "../common/src/runtime_context.cpp", "../common/src/runtime_context_impl.cpp", "../common/src/schema_object.cpp", @@ -118,6 +124,12 @@ ohos_source_set("src_file") { "../interfaces/src/kv_store_nb_delegate_impl.cpp", "../interfaces/src/kv_store_result_set_impl.cpp", "../interfaces/src/kv_store_snapshot_delegate_impl.cpp", + "../interfaces/src/relational/data_value.cpp", + "../interfaces/src/relational/relational_store_delegate_impl.cpp", + "../interfaces/src/relational/relational_store_manager.cpp", + "../interfaces/src/relational/relational_store_sqlite_ext.cpp", + "../interfaces/src/relational/runtime_config.cpp", + "../storage/src/data_transformer.cpp", "../storage/src/default_factory.cpp", "../storage/src/generic_kvdb.cpp", "../storage/src/generic_kvdb_connection.cpp", @@ -148,10 +160,16 @@ ohos_source_set("src_file") { "../storage/src/operation/multi_ver_database_oper.cpp", "../storage/src/operation/single_ver_database_oper.cpp", "../storage/src/package_file.cpp", + "../storage/src/relational_store_connection.cpp", + "../storage/src/relational_store_instance.cpp", + "../storage/src/relational_sync_able_storage.cpp", "../storage/src/result_entries_window.cpp", "../storage/src/single_ver_natural_store_commit_notify_data.cpp", "../storage/src/sqlite/query_object.cpp", "../storage/src/sqlite/query_sync_object.cpp", + "../storage/src/sqlite/relational/sqlite_relational_store.cpp", + "../storage/src/sqlite/relational/sqlite_relational_store_connection.cpp", + "../storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp", "../storage/src/sqlite/sqlite_local_kvdb.cpp", "../storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "../storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -165,6 +183,7 @@ ohos_source_set("src_file") { "../storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "../storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "../storage/src/sqlite/sqlite_single_ver_result_set.cpp", "../storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", "../storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", @@ -177,6 +196,7 @@ ohos_source_set("src_file") { "../storage/src/storage_engine.cpp", "../storage/src/storage_engine_manager.cpp", "../storage/src/storage_executor.cpp", + "../storage/src/sync_able_engine.cpp", "../storage/src/sync_able_kvdb.cpp", "../storage/src/sync_able_kvdb_connection.cpp", "../storage/src/upgrader/single_ver_database_upgrader.cpp", @@ -197,6 +217,7 @@ ohos_source_set("src_file") { "../syncer/src/single_ver_data_sync.cpp", "../syncer/src/single_ver_data_sync_with_sliding_window.cpp", "../syncer/src/single_ver_kv_syncer.cpp", + "../syncer/src/single_ver_relational_syncer.cpp", "../syncer/src/single_ver_serialize_manager.cpp", "../syncer/src/single_ver_sync_engine.cpp", "../syncer/src/single_ver_sync_state_machine.cpp", -- Gitee From 557de450c6a1edba2800e201f685a2e73966deaa Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 22 Dec 2021 15:12:19 +0800 Subject: [PATCH 03/53] remove callback in open relational store API Signed-off-by: lianhuix --- .../relational/relational_store_manager.h | 14 +++--- .../relational/relational_store_manager.cpp | 48 ++++++++++--------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 17f669b00..459196bdf 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -36,17 +36,17 @@ public: RelationalStoreManager &operator=(const RelationalStoreManager &) = delete; RelationalStoreManager &operator=(RelationalStoreManager &&) = delete; - // PRAGMA journal_mode=WAL - // PRAGMA synchronous=FULL - // PRAGMA synchronous=NORMAL - DB_API void OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, - const std::function &callback); + DB_API DBStatus OpenStore(const std::string &path, const std::string &storeId, + const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate); - DB_API DBStatus CloseStore(RelationalStoreDelegate *store); + DB_API DBStatus CloseStore(RelationalStoreDelegate *store); // TODO: move interface to delegate - DB_API DBStatus DeleteStore(const std::string &path); + DB_API DBStatus DeleteStore(const std::string &path); // TODO: remove interface private: + void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, + const std::string &storeId, RelationalDBProperties &properties); + std::string appId_; std::string userId_; }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 47bd5eb62..cb4fabae3 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -32,15 +32,15 @@ RelationalStoreManager::RelationalStoreManager(const std::string &appId, const s userId_(userId) {} -static void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, - const std::string &appId, const std::string &userId, RelationalDBProperties &properties) +void RelationalStoreManager::InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, + const std::string &storeId, RelationalDBProperties &properties) { properties.SetBoolProp(RelationalDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); - properties.SetStringProp(RelationalDBProperties::APP_ID, appId); - properties.SetStringProp(RelationalDBProperties::USER_ID, userId); - properties.SetStringProp(RelationalDBProperties::STORE_ID, storePath); // same as dir - std::string identifier = userId + "-" + appId + "-" + storePath; + properties.SetStringProp(RelationalDBProperties::APP_ID, appId_); + properties.SetStringProp(RelationalDBProperties::USER_ID, userId_); + properties.SetStringProp(RelationalDBProperties::STORE_ID, storeId); // same as dir + std::string identifier = userId_ + "-" + appId_ + "-" + storeId; std::string hashIdentifier = DBCommon::TransferHashString(identifier); properties.SetStringProp(RelationalDBProperties::IDENTIFIER_DATA, hashIdentifier); } @@ -64,39 +64,41 @@ static RelationalStoreConnection *GetOneConnectionWithRetry(const RelationalDBPr return nullptr; } -void RelationalStoreManager::OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, - const std::function &callback) +DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const std::string &storeId, + const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate) { - if (!callback) { - LOGE("[KvStoreMgr] Invalid callback for kv store!"); - return; + if (delegate != nullptr) { + LOGE("[RelationalStoreMgr] Invalid delegate!"); + return INVALID_ARGS; } - if (!ParamCheckUtils::CheckStoreParameter("Relational_default_id", appId_, userId_) || path.empty()) { - callback(INVALID_ARGS, nullptr); - return; + std::string canonicalDir; + if (!ParamCheckUtils::CheckDataDir(path, canonicalDir)) { + return INVALID_ARGS; + } + + if (!ParamCheckUtils::CheckStoreParameter(storeId, appId_, userId_) || path.empty()) { + return INVALID_ARGS; } RelationalDBProperties properties; - InitStoreProp(option, path, appId_, userId_, properties); + InitStoreProp(option, canonicalDir, storeId, properties); int errCode = E_OK; auto *conn = GetOneConnectionWithRetry(properties, errCode); DBStatus status = TransferDBErrno(errCode); if (conn == nullptr) { - callback(status, nullptr); - return; + return status; } - auto store = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); - if (store == nullptr) { + delegate = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); + if (delegate == nullptr) { conn->Close(); - callback(DB_ERROR, nullptr); - return; + return DB_ERROR; } - (void)conn->TriggerAutoSync(); - callback(OK, store); + (void)conn->TriggerAutoSync(); // TODO: no auto sync + return OK; } DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *store) -- Gitee From a4931b16b5f5adc2efe1362fb9789c8aaf3afd55 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 23 Dec 2021 09:39:26 +0800 Subject: [PATCH 04/53] Add read relational schema from meta Signed-off-by: lianhuix --- .../distributeddb/interfaces/include/types.h | 2 + .../storage/src/relational_store_instance.cpp | 14 ++-- .../relational/sqlite_relational_store.cpp | 73 ++++++++++++++++--- .../relational/sqlite_relational_store.h | 4 + ...qlite_single_relational_storage_engine.cpp | 7 -- 5 files changed, 77 insertions(+), 23 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h index fb12ab170..dca53003d 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h @@ -56,6 +56,8 @@ enum DBStatus { SECURITY_OPTION_CHECK_ERROR, // such as remote device's SecurityOption not equal to local SCHEMA_VIOLATE_VALUE, // Values already exist in dbFile do not match new schema INTERCEPT_DATA_FAIL, // Interceptor push data failed. + RELATIONAL_SCHEMA_NOT_FOUND, // the sync table is not a relational table + RELATIONAL_SCHEMA_CHANGED, // the schema was changed }; struct KvStoreConfig { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp index 9a304762e..366d78863 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp @@ -18,8 +18,9 @@ #include #include -#include "sqlite_relational_store.h" +#include "db_common.h" #include "db_errno.h" +#include "sqlite_relational_store.h" #include "log_print.h" namespace DistributedDB { @@ -65,7 +66,6 @@ static IRelationalStore *GetFromCache(const RelationalDBProperties &properties, } auto *db = iter->second; - if (db == nullptr) { LOGE("Store cache is nullptr, there may be a logic error"); errCode = -E_INTERNAL_ERROR; @@ -102,7 +102,7 @@ IRelationalStore *RelationalStoreInstance::OpenDatabase(const RelationalDBProper { auto db = new (std::nothrow) SQLiteRelationalStore(); if (db == nullptr) { - LOGE("Failed to get IKvDB! err:%d", errCode); + LOGE("Failed to get relational store! err:%d", errCode); return nullptr; } @@ -145,8 +145,11 @@ IRelationalStore *RelationalStoreInstance::GetDataBase(const RelationalDBPropert return db; } -RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const RelationalDBProperties &properties, int &errCode) +RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const RelationalDBProperties &properties, + int &errCode) { + std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + LOGD("Begin to get [%s] database connection.", STR_MASK(DBCommon::TransferStringToHex(identifier))); IRelationalStore *db = GetDataBase(properties, errCode); if (db == nullptr) { LOGE("Failed to open the db:%d", errCode); @@ -157,9 +160,8 @@ RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const if (connection == nullptr) { // not kill db, Other operations like import may be used concurrently LOGE("Failed to get the db connect for delegate:%d", errCode); } - RefObject::DecObjRef(db); // restore the reference increased by the cache. - // kvDB = nullptr; + RefObject::DecObjRef(db); // restore the reference increased by the cache. return connection; } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index dfaf56757..051369c0e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -15,12 +15,17 @@ #ifdef RELATIONAL_STORE #include "sqlite_relational_store.h" +#include "db_common.h" #include "db_errno.h" #include "log_print.h" #include "db_types.h" #include "sqlite_relational_store_connection.h" namespace DistributedDB { +namespace { + constexpr const char *RELATIONAL_SCHEMA_KEY = "relational_schema"; +} + SQLiteRelationalStore::~SQLiteRelationalStore() { delete sqliteStorageEngine_; @@ -39,7 +44,6 @@ RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) { std::lock_guard lock(connectMutex_); RelationalStoreConnection* connection = new (std::nothrow) SQLiteRelationalStoreConnection(this); - if (connection == nullptr) { errCode = -E_OUT_OF_MEMORY; return nullptr; @@ -68,24 +72,73 @@ int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &kvDBP return errCode; } -int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) +int SQLiteRelationalStore::GetSchemaFromMeta() { - sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); - if (sqliteStorageEngine_ == nullptr) { - LOGE("[RelationalStore] Create storage engine failed"); - return -E_OUT_OF_MEMORY; + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + int errCode = storageEngine_->GetMetaData(schemaKey, schemaVal); + if (errCode != E_OK && errCode != -E_NOT_FOUND ) { + LOGE("Get relationale schema from meta table failed. %d", errCode); + return errCode; + } else if (errCode == -E_NOT_FOUND) { + LOGW("No relational schema info was found."); + return E_OK; } - int errCode = InitStorageEngine(properties); + std::string schemaStr; + DBCommon::VectorToString(schemaVal, schemaStr); + RelationalSchemaObject schema; + errCode = schema.ParseFromSchemaString(schemaStr); if (errCode != E_OK) { - LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); + LOGE("Parse schema string from mata table failed."); return errCode; } - storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); - syncEngine_ = std::make_shared(storageEngine_); + + properties_.SetSchema(schema); return E_OK; } +int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) +{ + sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); + if (sqliteStorageEngine_ == nullptr) { + LOGE("[RelationalStore][Open] Create storage engine failed"); + return -E_OUT_OF_MEMORY; + } + + int errCode = E_OK; + + do { + errCode = InitStorageEngine(properties); + if (errCode != E_OK) { + LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); + break; + } + storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); + if (storageEngine_ == nullptr) { + LOGE("[RelationalStore][Open] Create syncable storage failed"); // TODO: + errCode = -E_OUT_OF_MEMORY; + break; + } + + properties_ = properties; + errCode = GetSchemaFromMeta(); + if (errCode != E_OK) { + break; + } + + // TODO: save log table version into meta data + + // TODO: clean the device table + + syncEngine_ = std::make_shared(storageEngine_); + return E_OK; + } while (false); + + // TODO: release resources. + return errCode; +} + void SQLiteRelationalStore::OnClose(const std::function ¬ifier) { AutoLock lockGuard(this); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 0a6f2ab68..9c06e28bf 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -54,6 +54,8 @@ private: // 1 store 1 connection void DecreaseConnectionCounter(); + int GetSchemaFromMeta(); + // use for sync Interactive std::shared_ptr syncEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv @@ -65,6 +67,8 @@ private: std::mutex connectMutex_; std::atomic connectionCount_ = 0; std::vector> closeNotifiers_; + + RelationalDBProperties properties_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index a47129d7c..6c9a5390f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -67,13 +67,6 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage break; } - // TODO: Get and parse relational schema from meta table - - // TODO: save log table version into meta data - - // TODO: clean the device table - - handle = NewSQLiteStorageExecutor(db, isWrite, false); if (handle == nullptr) { LOGE("[Relational] New SQLiteStorageExecutor[%d] for the pool failed.", isWrite); -- Gitee From 7630702b7e8d17d12b4d47af4b62bba4ebf94785 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 23 Dec 2021 10:29:12 +0800 Subject: [PATCH 05/53] Add save table version and clean device table. No Implement, Only Framework. Signed-off-by: lianhuix --- .../relational/sqlite_relational_store.cpp | 49 ++++++++++++++++++- .../relational/sqlite_relational_store.h | 6 +++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 051369c0e..83b3d7979 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -20,10 +20,13 @@ #include "log_print.h" #include "db_types.h" #include "sqlite_relational_store_connection.h" +#include "storage_engine_manager.h" namespace DistributedDB { namespace { constexpr const char *RELATIONAL_SCHEMA_KEY = "relational_schema"; + constexpr const char *LOG_TABLE_VERSION_KEY = "log_table_versoin"; + constexpr const char *LOG_TABLE_VERSION_1 = "1.0"; } SQLiteRelationalStore::~SQLiteRelationalStore() @@ -72,6 +75,15 @@ int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &kvDBP return errCode; } +void SQLiteRelationalStore::ReleaseResources() +{ + // TODO: Lock + if (sqliteStorageEngine_ != nullptr) { + sqliteStorageEngine_->ClearEnginePasswd(); + (void)StorageEngineManager::ReleaseStorageEngine(sqliteStorageEngine_); + } +} + int SQLiteRelationalStore::GetSchemaFromMeta() { const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); @@ -98,6 +110,32 @@ int SQLiteRelationalStore::GetSchemaFromMeta() return E_OK; } +int SQLiteRelationalStore::SaveLogTableVersionToMeta() +{ + // TODO: save log table version into meta data + LOGD("save log table version to meta table, key: %s, val: %s", LOG_TABLE_VERSION_KEY, LOG_TABLE_VERSION_1); + return E_OK; +} + +int SQLiteRelationalStore::CleanDistributedDeviceTable() +{ + // TODO: clean the device table which is no longer in schema + RelationalSchemaObject schema = properties_.GetSchema(); + for (const auto &table : schema.GetTables()) { + std::string tableName = table.first; + LOGD("Get schema %s.", tableName.c_str()); + } + + int errCode = E_OK; + auto *handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + // TODO: Get device table names, and clean + ReleaseHandle(handle); + return E_OK; +} + int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) { sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); @@ -127,15 +165,22 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } - // TODO: save log table version into meta data + errCode = SaveLogTableVersionToMeta(); + if (errCode != E_OK) { + break; + } - // TODO: clean the device table + errCode = CleanDistributedDeviceTable(); + if (errCode != E_OK) { + break; + } syncEngine_ = std::make_shared(storageEngine_); return E_OK; } while (false); // TODO: release resources. + ReleaseResources(); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 9c06e28bf..e927e2aa2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -51,11 +51,17 @@ public: return storageEngine_; } private: + void ReleaseResources(); + // 1 store 1 connection void DecreaseConnectionCounter(); int GetSchemaFromMeta(); + int SaveLogTableVersionToMeta(); + + int CleanDistributedDeviceTable(); + // use for sync Interactive std::shared_ptr syncEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv -- Gitee From e7315a1ad40bddc403c5b966f84710ffc92a9735 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 23 Dec 2021 15:08:10 +0800 Subject: [PATCH 06/53] Add create distributed table implement Signed-off-by: lianhuix --- .../common/include/db_constant.h | 2 ++ .../common/include/param_check_utils.h | 2 ++ .../distributeddb/common/src/db_constant.cpp | 2 ++ .../common/src/param_check_utils.cpp | 5 ++++ .../relational/relational_schema_object.cpp | 6 +++- .../relational/relational_store_delegate.h | 5 +--- .../relational_store_delegate_impl.cpp | 13 ++++++--- .../relational_store_delegate_impl.h | 2 +- .../include/relational_store_connection.h | 3 +- .../relational/sqlite_relational_store.cpp | 28 +++++++++++++++++++ .../relational/sqlite_relational_store.h | 2 ++ .../sqlite_relational_store_connection.cpp | 25 ++++++++--------- .../sqlite_relational_store_connection.h | 3 +- ...single_ver_relational_storage_executor.cpp | 8 +++--- ...e_single_ver_relational_storage_executor.h | 4 ++- 15 files changed, 77 insertions(+), 33 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index 99a6944b9..722df4437 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -92,6 +92,8 @@ public: static const std::string UPDATE_META_FUNC; + static const std::string SYSTEM_TABLE_PREFIX; + static constexpr uint32_t AUTO_SYNC_TIMEOUT = 5000; // 5s static constexpr uint32_t MANUAL_SYNC_TIMEOUT = 5000; // 5s diff --git a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h index 0e4c0dbc8..ce07d65e1 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h @@ -49,6 +49,8 @@ public: SchemaObject &schemaObject, std::string &canonicalDir); static uint8_t GetValidCompressionRate(uint8_t compressionRate); + + static bool CheckRelationalTableName(const std::string &tableName); }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp index 410e01137..269b46436 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp @@ -63,4 +63,6 @@ const std::string DBConstant::TRIGGER_REFERENCES_NEW = "NEW."; const std::string DBConstant::TRIGGER_REFERENCES_OLD = "OLD."; const std::string DBConstant::UPDATE_META_FUNC = "update_meta_within_trigger"; + +const std::string DBConstant::SYSTEM_TABLE_PREFIX = "naturalbase_rdb_"; } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp b/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp index e313bd4b4..58ee11566 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp @@ -187,4 +187,9 @@ uint8_t ParamCheckUtils::GetValidCompressionRate(uint8_t compressionRate) } return compressionRate; } + +bool ParamCheckUtils::CheckRelationalTableName(const std::string &tableName) +{ + return tableName.compare(0, DBConstant::SYSTEM_TABLE_PREFIX.size(), DBConstant::SYSTEM_TABLE_PREFIX) != 0; +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index 2c1ecbf67..ef1578e66 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -425,7 +425,11 @@ const std::map &RelationalSchemaObject::GetTables() cons const TableInfo &RelationalSchemaObject::GetTable(const std::string& tableName) const { - return tables_.at(tableName); + auto it = tables_.find(tableName); + if (it != tables_.end()) { + return it->second; + } + return {}; } int RelationalSchemaObject::CompareAgainstSchemaObject(const std::string &inSchemaString, diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 3e2923b77..6dc424277 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -34,10 +34,7 @@ public: DB_API virtual DBStatus RemoveDeviceData(const std::string &device) = 0; - struct TableOption { - }; - - DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) = 0; + DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName) = 0; DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index fb4d08542..c9c1b842e 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -16,8 +16,9 @@ #include "relational_store_delegate_impl.h" #include "db_errno.h" -#include "log_print.h" #include "kv_store_errno.h" +#include "log_print.h" +#include "param_check_utils.h" #include "sync_operation.h" namespace DistributedDB { @@ -52,15 +53,19 @@ DBStatus RelationalStoreDelegateImpl::RemoveDeviceData(const std::string &device return NOT_SUPPORT; } -DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName, const TableOption &option) +DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName) { - // check table Name and option + if (!ParamCheckUtils::CheckRelationalTableName(tableName)) { + LOGE("invalid table name."); + return INVALID_ARGS; + } + if (conn_ == nullptr) { LOGE("[RelationalStore Delegate] Invalid connection for operation!"); return DB_ERROR; } - int errCode = conn_->CreateDistributedTable(tableName, option); + int errCode = conn_->CreateDistributedTable(tableName); if (errCode != E_OK) { LOGW("[RelationalStore Delegate] Create Distributed table failed:%d", errCode); return TransferDBErrno(errCode); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 25d51d440..25d7e6c96 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -39,7 +39,7 @@ public: DBStatus RemoveDeviceData(const std::string &device) override; - DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) override; + DBStatus CreateDistributedTable(const std::string &tableName) override; DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) override; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index e502357a7..35f2d91e6 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -49,8 +49,7 @@ public: virtual int TriggerAutoSync() = 0; virtual int SyncToDevice(SyncInfo &info) = 0; virtual std::string GetIdentifier() = 0; - virtual int CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) = 0; + virtual int CreateDistributedTable(const std::string &tableName) = 0; protected: // Get the stashed 'KvDB_ pointer' without ref. diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 83b3d7979..51b45a926 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -276,5 +276,33 @@ void SQLiteRelationalStore::WakeUpSyncer() { syncEngine_->WakeUpSyncer(); } + + +int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) +{ + int errCode = E_OK; + auto schema = properties_.GetSchema(); + if (schema.GetTable(tableName).GetTableName() == tableName) { + LOGW("distributed table %s was already created.", tableName.c_str()); + return E_OK; + } + + auto *handle = GetHandle(true, errCode); + if (handle != nullptr) { + return errCode; + } + + TableInfo table; + errCode = handle->CreateDistributedTable(tableName, table); + if (errCode != E_OK) { + LOGE("create distributed table failed. %d", errCode); + } else { + schema.AddRelationalTable(table); + properties_.SetSchema(schema); + } + + ReleaseHandle(handle); + return E_OK; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index e927e2aa2..5358dc5d7 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -50,6 +50,8 @@ public: { return storageEngine_; } + + int CreateDistributedTable(const std::string &tableName); private: void ReleaseResources(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index a66269b12..bc86d438c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -58,7 +58,7 @@ SQLiteSingleVerRelationalStorageExecutor *SQLiteRelationalStoreConnection::GetEx auto *store = GetDB(); if (store == nullptr) { errCode = -E_NOT_INIT; - LOGE("[SingleVerConnection] store is null, get executor failed! errCode = [%d]", errCode); + LOGE("[RelationalConnection] store is null, get executor failed! errCode = [%d]", errCode); return nullptr; } @@ -93,7 +93,7 @@ int SQLiteRelationalStoreConnection::StartTransaction() return errCode; } - LOGD("[SingleVerConnection] Start transaction finish."); + LOGD("[RelationalConnection] Start transaction finish."); writeHandle_ = handle; transactingFlag_.store(true); return E_OK; @@ -135,20 +135,17 @@ int SQLiteRelationalStoreConnection::RollBack() return errCode; } -int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) +int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &tableName) { - int errCode = StartTransaction(); - if (errCode != E_OK && errCode != E_TRANSACT_STATE) { - return errCode; + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[RelationalConnection] store is null, get DB failed!"); + return -E_INVALID_CONNECTION; } - errCode = writeHandle_->CreateDistributedTable(tableName, option); - if (errCode == E_OK) { - errCode = Commit(); - } else { - int innerCode = RollBack(); - errCode = (innerCode != E_OK) ? innerCode : errCode; + int errCode = store->CreateDistributedTable(tableName); + if (errCode != E_OK) { + LOGE("[RelationalConnection] crete distributed table failed. %d", errCode); } return errCode; } @@ -167,7 +164,7 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) { auto *store = GetDB(); if (store == nullptr) { - LOGE("[SingleVerConnection] store is null, get executor failed!"); + LOGE("[RelationalConnection] store is null, get executor failed!"); return -E_INVALID_CONNECTION; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 020560850..63a533f5f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -37,8 +37,7 @@ public: int TriggerAutoSync() override; int SyncToDevice(SyncInfo &info) override; std::string GetIdentifier() override; - int CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) override; + int CreateDistributedTable(const std::string &tableName) override; protected: diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index be162f3b0..17bd91e9d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -19,17 +19,16 @@ namespace DistributedDB { SQLiteSingleVerRelationalStorageExecutor::SQLiteSingleVerRelationalStorageExecutor(sqlite3 *dbHandle, bool writable) - : SQLiteStorageExecutor(dbHandle, writable, false) {}; + : SQLiteStorageExecutor(dbHandle, writable, false) +{} -int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) +int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, TableInfo &table) { if (dbHandle_ == nullptr) { LOGE("Begin transaction failed, dbHandle is null."); return -E_INVALID_DB; } - TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode != E_OK) { LOGE("[CreateDistributedTable] analysis table schema failed"); @@ -98,6 +97,7 @@ int SQLiteSingleVerRelationalStorageExecutor::Rollback() int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) { + // TODO: Get table info from schema int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); if (errCode != E_OK) { LOGE("[CreateDistributedTable] analysis table schema failed"); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h index 301204f61..1ab02ab3b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -32,7 +32,7 @@ public: // Delete the copy and assign constructors DISABLE_COPY_ASSIGN_MOVE(SQLiteSingleVerRelationalStorageExecutor); - int CreateDistributedTable(const std::string &tableName, const RelationalStoreDelegate::TableOption &option); + int CreateDistributedTable(const std::string &tableName, TableInfo &table); int StartTransaction(TransactType type); int Commit(); @@ -57,6 +57,8 @@ public: int SaveSyncItems(const QueryObject &object, std::vector &dataItems, const std::string &deviceName, TimeStamp &timeStamp); + int AnalysisRelationalSchema(const std::string &tableName, TableInfo &tableInfo); + private: int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData) const; -- Gitee From a87bf3cc7bc2c7535aef5eed6c8e2deff1f0543b Mon Sep 17 00:00:00 2001 From: lianhuix Date: Fri, 24 Dec 2021 09:25:07 +0800 Subject: [PATCH 07/53] Add relational schema field attribute parse. Signed-off-by: lianhuix --- .../common/include/json_object.h | 2 + .../relational/relational_schema_object.h | 1 + .../distributeddb/common/src/json_object.cpp | 20 +++++ .../relational/relational_schema_object.cpp | 77 ++++++++++++------- .../relational/relational_store_delegate.h | 2 +- .../relational/relational_store_manager.cpp | 4 +- .../relational/sqlite_relational_store.cpp | 15 +++- .../relational/sqlite_relational_store.h | 3 + .../storage/src/sync_able_engine.cpp | 3 +- 9 files changed, 89 insertions(+), 38 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h index c9ecd26e6..0fcaf6478 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h @@ -65,6 +65,8 @@ public: int GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; int GetStringArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; + int GetObjectByFieldPath(const FieldPath &inPath, JsonObject &outArray) const; + // An empty fieldPath indicate the root, the outSubPath should be empty before call, we will not empty it at first. // If inPath is of multiple path, then outSubPath is combination of result of each inPath. int GetSubFieldPath(const FieldPath &inPath, std::set &outSubPath) const; diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index 47e4f99aa..c91a1d711 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -153,6 +153,7 @@ private: int ParseCheckTableInfo(const JsonObject &inJsonObject); int ParseCheckTableName(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableDefine(const JsonObject &inJsonObject, TableInfo &resultTable); + int ParseCheckTableFieldInfo(const JsonObject &inJsonObject, const FieldPath &path, FieldInfo &table); int ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableIndex(const JsonObject &inJsonObject, TableInfo &resultTable); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index 37a008ddb..80313347a 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -741,6 +741,26 @@ int JsonObject::GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const { if (!isValid_) { diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index ef1578e66..5aa861f8c 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -97,15 +97,13 @@ void FieldInfo::SetColumnId(int cid) // return field define string like ("fieldName": "MY INT(21), NOT NULL, DEFAULT 123") std::string FieldInfo::ToAttributeString() const { - std::string attrStr = "\"" + fieldName_ + "\": \""; - attrStr += dataType_; - if (isNotNull_) { - attrStr += ", NOT NULL"; - } + std::string attrStr = "\"" + fieldName_ + "\": {"; + attrStr += "\"TYPE\":\"" + dataType_ + "\","; + attrStr += "\"NOT_NULL\":" + std::string(isNotNull_ ? "true" : "false") + ","; if (hasDefaultValue_) { - attrStr += ", " + defaultValue_; + attrStr += "\"DEFAULT\":\"" + defaultValue_ + "\""; } - attrStr += + "\""; + attrStr += "}"; return attrStr; } @@ -448,8 +446,8 @@ namespace { int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &fieldName, FieldType expectType, bool isNecessary, FieldValue &fieldValue) { - if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName}) && !isNecessary) { - return E_OK; + if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName})) { + return isNecessary ? -E_SCHEMA_PARSE_FAIL : -E_NOT_FOUND; } FieldType fieldType; @@ -600,46 +598,67 @@ int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject } for (const auto &field : tableFields) { - if (field.second != FieldType::LEAF_FIELD_STRING) { - LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType STRING but : %s.", + if (field.second != FieldType::LEAF_FIELD_OBJECT) { + LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType OBJECT but : %s.", SchemaUtils::FieldTypeString(field.second).c_str()); return -E_SCHEMA_PARSE_FAIL; } - FieldValue fieldValue; - errCode = inJsonObject.GetFieldValueByFieldPath(field.first, fieldValue); - if (errCode != E_OK) { - LOGE("[RelationalSchema][Parse] Get schema TABLES DEFINE field value failed: %d.", errCode); - return -E_SCHEMA_PARSE_FAIL; - } - SchemaAttribute outAttr; - errCode = SchemaUtils::ParseAndCheckSchemaAttribute(fieldValue.stringValue, outAttr, false); + JsonObject fieldObj; + errCode = inJsonObject.GetObjectByFieldPath(field.first, fieldObj); if (errCode != E_OK) { - LOGE("[RelationalSchema][Parse] Parse schema TABLES DEFINE attribute failed: %d.", errCode); + LOGE("[RelationalSchema][Parse] Get table field object failed. %s", errCode); return errCode; } FieldInfo fieldInfo; - fieldInfo.SetFieldName(field.first[1]); - fieldInfo.SetDataType(outAttr.customFieldType); - fieldInfo.SetNotNull(outAttr.hasNotNullConstraint); - if (outAttr.hasDefaultValue) { - fieldInfo.SetDefaultValue(outAttr.defaultValue.stringValue); + fieldInfo.SetFieldName(field.first[0]); // 0 : first element in path + errCode = ParseCheckTableFieldInfo(fieldObj, field.first, fieldInfo); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Parse table field info failed. %d", errCode); + return -E_SCHEMA_PARSE_FAIL; } resultTable.AddField(fieldInfo); } return E_OK; } +int RelationalSchemaObject::ParseCheckTableFieldInfo(const JsonObject &inJsonObject, const FieldPath &path, + FieldInfo &table) +{ + FieldValue fieldValue; + int errCode = GetMemberFromJsonObject(inJsonObject, "TYPE", FieldType::LEAF_FIELD_STRING, true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + table.SetDataType(fieldValue.stringValue); + + errCode = GetMemberFromJsonObject(inJsonObject, "NOT_NULL", FieldType::LEAF_FIELD_BOOL, true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + table.SetNotNull(fieldValue.boolValue); + + errCode = GetMemberFromJsonObject(inJsonObject, "DEFAULT", FieldType::LEAF_FIELD_STRING, false, fieldValue); + if (errCode == E_OK) { + table.SetDefaultValue(fieldValue.stringValue); + } else if (errCode != -E_NOT_FOUND) { + return errCode; + } + // TODO: need cid or not? + return E_OK; +} + int RelationalSchemaObject::ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable) { FieldValue fieldValue; - int errCode = GetMemberFromJsonObject(inJsonObject, "AUTOINCREMENT", FieldType::LEAF_FIELD_BOOL, - false, fieldValue); + int errCode = GetMemberFromJsonObject(inJsonObject, "AUTOINCREMENT", FieldType::LEAF_FIELD_BOOL, false, fieldValue); if (errCode == E_OK) { resultTable.SetAutoIncrement(fieldValue.boolValue); + } else if (errCode != -E_NOT_FOUND) { + return errCode; } - return errCode; + return E_OK; } int RelationalSchemaObject::ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable) @@ -685,7 +704,7 @@ int RelationalSchemaObject::ParseCheckTableIndex(const JsonObject &inJsonObject, LOGE("[RelationalSchema][Parse] Get schema TABLES INDEX field value failed: %d.", errCode); return -E_SCHEMA_PARSE_FAIL; } - resultTable.AddIndexDefine(field.first[1], indexDefine); + resultTable.AddIndexDefine(field.first[1], indexDefine); // 1 : second element in path } return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 6dc424277..1b7f3d3ef 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -27,7 +27,7 @@ public: DB_API virtual ~RelationalStoreDelegate() = default; struct Option { - bool createIfNecessary = true; + // split mode }; DB_API virtual DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index cb4fabae3..a936be02a 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -35,7 +35,7 @@ RelationalStoreManager::RelationalStoreManager(const std::string &appId, const s void RelationalStoreManager::InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, const std::string &storeId, RelationalDBProperties &properties) { - properties.SetBoolProp(RelationalDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); + // properties.SetBoolProp(RelationalDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); properties.SetStringProp(RelationalDBProperties::APP_ID, appId_); properties.SetStringProp(RelationalDBProperties::USER_ID, userId_); @@ -96,8 +96,6 @@ DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const conn->Close(); return DB_ERROR; } - - (void)conn->TriggerAutoSync(); // TODO: no auto sync return OK; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 51b45a926..b689a3737 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -31,7 +31,10 @@ namespace { SQLiteRelationalStore::~SQLiteRelationalStore() { - delete sqliteStorageEngine_; + if (sqliteStorageEngine_ != nullptr) { + delete sqliteStorageEngine_; + sqliteStorageEngine_ = nullptr; + } } // Called when a new connection created. @@ -138,6 +141,12 @@ int SQLiteRelationalStore::CleanDistributedDeviceTable() int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) { + std::lock_guard lock(initalMutex_); + if (isInitialized_) { + LOGD("[RelationalStore][Open] relational db was already inited."); + return E_OK; + } + sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); if (sqliteStorageEngine_ == nullptr) { LOGE("[RelationalStore][Open] Create storage engine failed"); @@ -145,13 +154,13 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) } int errCode = E_OK; - do { errCode = InitStorageEngine(properties); if (errCode != E_OK) { LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); break; } + storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); if (storageEngine_ == nullptr) { LOGE("[RelationalStore][Open] Create syncable storage failed"); // TODO: @@ -176,10 +185,10 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) } syncEngine_ = std::make_shared(storageEngine_); + isInitialized_ = true; return E_OK; } while (false); - // TODO: release resources. ReleaseResources(); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 5358dc5d7..8c7b6127f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -77,6 +77,9 @@ private: std::vector> closeNotifiers_; RelationalDBProperties properties_; + + mutable std::mutex initalMutex_; + bool isInitialized_ = false; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp index 5ae456f8f..f2ca51301 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -26,8 +26,7 @@ SyncAbleEngine::SyncAbleEngine(ISyncInterface *store) {} SyncAbleEngine::~SyncAbleEngine() -{ -} +{} // Start a sync action. int SyncAbleEngine::Sync(const ISyncer::SyncParma &parm) -- Gitee From da2c4800627a3d63194e7f9b3476df72701879c9 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Fri, 24 Dec 2021 14:36:36 +0800 Subject: [PATCH 08/53] Fix relational schema issue. Signed-off-by: lianhuix --- .../relational/relational_schema_object.h | 2 +- .../distributeddb/common/src/json_object.cpp | 1 - .../relational/relational_schema_object.cpp | 13 +++++---- ...single_ver_relational_storage_executor.cpp | 28 +++++++++---------- .../storage/src/sqlite/sqlite_utils.cpp | 16 +++++------ 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index c91a1d711..ed59af074 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -138,7 +138,7 @@ public: const std::map &GetTables() const; - const TableInfo &GetTable(const std::string& tableName) const; + TableInfo GetTable(const std::string& tableName) const; private: int CompareAgainstSchemaObject(const std::string &inSchemaString, std::map &cmpRst) const; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index 80313347a..cb6c3516f 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -734,7 +734,6 @@ int JsonObject::GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &RelationalSchemaObject::GetTables() cons return tables_; } -const TableInfo &RelationalSchemaObject::GetTable(const std::string& tableName) const +TableInfo RelationalSchemaObject::GetTable(const std::string& tableName) const { auto it = tables_.find(tableName); if (it != tables_.end()) { @@ -447,6 +447,7 @@ int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &f bool isNecessary, FieldValue &fieldValue) { if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName})) { + LOGW("[RelationalSchema][Parse] Get schema %s not exist. isNeccessary: %d", fieldName.c_str(), isNecessary); return isNecessary ? -E_SCHEMA_PARSE_FAIL : -E_NOT_FOUND; } @@ -458,11 +459,11 @@ int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &f } if (fieldType != expectType) { - LOGE("[RelationalSchema][Parse] Expect %s fieldType %d but : %d.", fieldName.c_str(), expectType, fieldType); + LOGE("[RelationalSchema][Parse] Expect %s fieldType %d but: %d.", fieldName.c_str(), expectType, fieldType); return -E_SCHEMA_PARSE_FAIL; } - errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {"NAME"}, fieldValue); + errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {fieldName}, fieldValue); if (errCode != E_OK) { LOGE("[RelationalSchema][Parse] Get schema %s value failed: %d.", fieldName.c_str(), errCode); return -E_SCHEMA_PARSE_FAIL; @@ -598,8 +599,8 @@ int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject } for (const auto &field : tableFields) { - if (field.second != FieldType::LEAF_FIELD_OBJECT) { - LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType OBJECT but : %s.", + if (field.second != FieldType::INTERNAL_FIELD_OBJECT) { + LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType INTERNAL OBJECT but : %s.", SchemaUtils::FieldTypeString(field.second).c_str()); return -E_SCHEMA_PARSE_FAIL; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 17bd91e9d..489e70eb2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -117,10 +117,10 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSyncDataByTime(TimeStamp } const std::string SELECT_SYNC_DELETED_ENTRIES_SQL = - "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + + "SELECT * FROM naturalbase_rdb_aux_" + table_.GetTableName() + "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x03) ORDER BY timestamp ASC;"; const std::string SELECT_SYNC_ENTRIES_SQL = - "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + + "SELECT * FROM naturalbase_rdb_aux_" + table_.GetTableName() + "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x02=0x02) ORDER BY timestamp ASC;"; const std::string sql = (getDeletedData ? SELECT_SYNC_DELETED_ENTRIES_SQL : SELECT_SYNC_ENTRIES_SQL); @@ -395,8 +395,7 @@ int SQLiteSingleVerRelationalStorageExecutor::GetDeletedSyncDataByTimestamp(std: return CheckCorruptedStatus(errCode); } -static const std::string SELECT_META_VALUE_SQL = - "SELECT value FROM distributeddatamgr_aux_metadata WHERE key=?;"; +static const std::string SELECT_META_VALUE_SQL = "SELECT value FROM naturalbase_rdb_aux_metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &value) const { sqlite3_stmt *statement = nullptr; @@ -424,7 +423,7 @@ int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &v return errCode; } -static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO distributeddatamgr_aux_metadata VALUES(?,?);"; +static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO naturalbase_rdb_aux_metadata VALUES(?,?);"; int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const { sqlite3_stmt *statement = nullptr; @@ -453,7 +452,7 @@ ERROR: return errCode; } -static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM distributeddatamgr_aux_metadata WHERE key=?;"; +static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM naturalbase_rdb_aux_metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const { sqlite3_stmt *statement = nullptr; @@ -480,7 +479,7 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector=? AND key<=?;"; + "DELETE FROM naturalbase_rdb_aux_metadata WHERE key>=? AND key<=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const { sqlite3_stmt *statement = nullptr; @@ -527,8 +526,7 @@ static int GetAllKeys(sqlite3_stmt *statement, std::vector &keys) return errCode; } - -static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM distributeddatamgr_aux_metadata;"; +static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM naturalbase_rdb_aux_metadata;"; int SQLiteSingleVerRelationalStorageExecutor::GetAllMetaKeys(std::vector &keys) const { sqlite3_stmt *statement = nullptr; @@ -546,7 +544,7 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj const std::string &deviceName, sqlite3_stmt *&logStmt) const { std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_log"; + const std::string tableName = "naturalbase_rdb_aux_" + object.GetTableName() + "_log"; std::string dataFormat = "?, '" + deviceName + "', ?, ?, ?, ?, ?"; std::string sql = "INSERT OR REPLACE INTO " + tableName + @@ -562,10 +560,10 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const { - // distributeddatamgr_aux_userTableName_deviceHash + // naturalbase_rdb_aux_userTableName_deviceHash // tableName std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_" + + const std::string tableName = "naturalbase_rdb_aux_" + object.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); @@ -603,7 +601,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen Key hashKey; (void)DBCommon::CalcValueHash(dataItem.key, hashKey); std::string hash = std::string(hashKey.begin(), hashKey.end()); - std::string sql = "select * from distributeddatamgr_aux_" + table_.GetTableName() + "_log where hash_key = ?;"; + std::string sql = "select * from naturalbase_rdb_aux_" + table_.GetTableName() + "_log where hash_key = ?;"; sqlite3_stmt *queryStmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, queryStmt); if (errCode != E_OK) { @@ -661,7 +659,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem) { std::string devName = DBCommon::TransferHashString(dataItem.dev); - const std::string tableName = "distributeddatamgr_aux_" + table_.GetTableName() + "_" + + const std::string tableName = "naturalbase_rdb_aux_" + table_.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); std::string hashKey = std::string(dataItem.hashKey.begin(), dataItem.hashKey.end()); std::string sql = "DELETE FROM " + tableName + " WHERE calc_hash(" + table_.GetPrimaryKey() + ")=" + hashKey + ";"; @@ -784,7 +782,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncItems(const QueryObject &o static int GetLogInfoStatement(sqlite3 *dbHandle, const std::string &tableName, uint64_t beginTime, uint64_t endTime, sqlite3_stmt *&statement) { - std::string sql = "select * from distributeddatamgr_aux_" + tableName + + std::string sql = "select * from naturalbase_rdb_aux_" + tableName + "_log where flag=0x02 AND timestamp>=? AND timestamp Date: Sat, 25 Dec 2021 11:40:25 +0800 Subject: [PATCH 09/53] Add create distributed table constraint. Signed-off-by: lianhuix --- .../common/include/db_constant.h | 1 + .../relational/sqlite_relational_store.cpp | 8 +++++++ .../relational/sqlite_relational_store.h | 2 ++ ...single_ver_relational_storage_executor.cpp | 16 +++++++++++-- .../storage/src/sqlite/sqlite_utils.cpp | 23 +++++++++++++++++++ .../storage/src/sqlite/sqlite_utils.h | 2 ++ 6 files changed, 50 insertions(+), 2 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index 722df4437..9c5d0e65b 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -111,6 +111,7 @@ public: static constexpr size_t MAX_SYNC_BLOCK_SIZE = 31457280; // 30MB static constexpr int DOUBLE_PRECISION = 15; + static constexpr int MAX_DISTRIBUTED_TABLE_COUNT = 32; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index b689a3737..31a26d5f6 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -109,6 +109,7 @@ int SQLiteRelationalStore::GetSchemaFromMeta() return errCode; } + std::lock_guard lock(schemaMutex_); properties_.SetSchema(schema); return E_OK; } @@ -123,6 +124,7 @@ int SQLiteRelationalStore::SaveLogTableVersionToMeta() int SQLiteRelationalStore::CleanDistributedDeviceTable() { // TODO: clean the device table which is no longer in schema + std::lock_guard lock(schemaMutex_); RelationalSchemaObject schema = properties_.GetSchema(); for (const auto &table : schema.GetTables()) { std::string tableName = table.first; @@ -290,12 +292,18 @@ void SQLiteRelationalStore::WakeUpSyncer() int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) { int errCode = E_OK; + std::lock_guard lock(schemaMutex_); auto schema = properties_.GetSchema(); if (schema.GetTable(tableName).GetTableName() == tableName) { LOGW("distributed table %s was already created.", tableName.c_str()); return E_OK; } + if (schema.GetTables().size() >= DBConstant::MAX_DISTRIBUTED_TABLE_COUNT) { + LOGW("The number of distributed tables is exceeds limit."); + return -E_MAX_LIMITS; + } + auto *handle = GetHandle(true, errCode); if (handle != nullptr) { return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 8c7b6127f..1d069659f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -80,6 +80,8 @@ private: mutable std::mutex initalMutex_; bool isInitialized_ = false; + + mutable std::mutex schemaMutex_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 489e70eb2..57e6a7181 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -25,11 +25,23 @@ SQLiteSingleVerRelationalStorageExecutor::SQLiteSingleVerRelationalStorageExecut int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, TableInfo &table) { if (dbHandle_ == nullptr) { - LOGE("Begin transaction failed, dbHandle is null."); + LOGE("[CreateDistributedTable] Begin transaction failed, dbHandle is null."); return -E_INVALID_DB; } - int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); + int historyDataCnt = 0; + int errCode = SQLiteUtils::GetTableCount(dbHandle_, tableName, historyDataCnt); + if (errCode != E_OK) { + LOGE("[CreateDistributedTable] Get the number of table [%s] rows failed. %d", tableName.c_str(), errCode); + return errCode; + } + + if (historyDataCnt > 0) { // 0 : create distributed table should on an empty table + LOGE("[CreateDistributedTable] Create distributed table should on an empty table."); + return -E_NOT_SUPPORT; + } + + errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode != E_OK) { LOGE("[CreateDistributedTable] analysis table schema failed"); return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 1dd2b58c4..32f30f031 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1921,4 +1921,27 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) sqlite3_free(eSql); return E_OK; } + +int GetTableCount(sqlite3 *db, const std::string &tableName, int &count) +{ + if (db == nullptr) { + return -E_INVALID_ARGS; + } + + std::string cntSql = "SELECT COUNT(*) FROM " + tableName + ";"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, cntSql, stmt); + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + count = sqlite3_column_int(stmt, 0); + errCode = E_OK; + } + + SQLiteUtils::ResetStatement(stmt, true, errCode); + return SQLiteUtils::MapSQLiteErrno(errCode); +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 94d714001..d8a9d8d01 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -174,6 +174,8 @@ public: static int ExpandedSql(sqlite3_stmt *stmt, std::string &basicString); + static int GetTableCount(sqlite3 *db, const std::string &tableName, int &count); + private: static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp); -- Gitee From 18c9db5c22fcdd441f5228ed7c8d3fe57eb67922 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Sat, 25 Dec 2021 15:01:25 +0800 Subject: [PATCH 10/53] Fix issues. Signed-off-by: lianhuix --- .../common/src/relational/relational_schema_object.cpp | 2 +- .../src/sqlite/relational/sqlite_relational_store.cpp | 5 +++-- .../libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp | 2 +- .../syncer/distributeddb_relational_ver_p2p_sync_test.cpp | 4 ++-- .../test/unittest/common/syncer/virtual_device.cpp | 4 ++-- .../test/unittest/common/syncer/virtual_device.h | 2 +- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index e3a3551ee..e5f33dab1 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -447,7 +447,7 @@ int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &f bool isNecessary, FieldValue &fieldValue) { if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName})) { - LOGW("[RelationalSchema][Parse] Get schema %s not exist. isNeccessary: %d", fieldName.c_str(), isNecessary); + LOGW("[RelationalSchema][Parse] Get schema %s not exist. isNecessary: %d", fieldName.c_str(), isNecessary); return isNecessary ? -E_SCHEMA_PARSE_FAIL : -E_NOT_FOUND; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 31a26d5f6..b3f7e305a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -304,8 +304,9 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) return -E_MAX_LIMITS; } + LOGD("Create distributed table for %s.", tableName.c_str()); auto *handle = GetHandle(true, errCode); - if (handle != nullptr) { + if (handle == nullptr) { return errCode; } @@ -319,7 +320,7 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) } ReleaseHandle(handle); - return E_OK; + return errCode; } } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 32f30f031..b900d4779 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1922,7 +1922,7 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) return E_OK; } -int GetTableCount(sqlite3 *db, const std::string &tableName, int &count) +int SQLiteUtils::GetTableCount(sqlite3 *db, const std::string &tableName, int &count) { if (db == nullptr) { return -E_INVALID_ARGS; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index c24f02a1e..1f76b576f 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -185,7 +185,7 @@ namespace { EXPECT_EQ(GetDB(db), SQLITE_OK); EXPECT_EQ(CreateTable(db), SQLITE_OK); - EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName, {}), OK); + EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName), OK); sqlite3_close(db); @@ -254,7 +254,7 @@ void DistributedDBRelationalVerP2PSyncTest::SetUp(void) /** * @tc.setup: create virtual device B, and get a KvStoreNbDelegate as deviceA */ - g_mgr.OpenStore(g_dbDir, {true}, g_kvDelegateCallback); + g_kvDelegateStatus = g_mgr.OpenStore(g_dbDir, "Relational_default_id", {}, g_kvDelegatePtr); ASSERT_TRUE(g_kvDelegateStatus == OK); ASSERT_TRUE(g_kvDelegatePtr != nullptr); g_deviceB = new (std::nothrow) RelationalVirtualDevice(DEVICE_B); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp index eaa1f7764..25749ee91 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp @@ -20,7 +20,7 @@ #include "device_manager.h" #include "log_print.h" #include "multi_ver_sync_state_machine.h" -#include "single_ver_subscribe_manager.h" +#include "subscribe_manager.h" #include "single_ver_sync_state_machine.h" #include "sync_types.h" #include "virtual_communicator.h" @@ -99,7 +99,7 @@ int VirtualDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggrega } if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_SVD) { context_ = new (std::nothrow) SingleVerSyncTaskContext; - subManager_ = std::make_shared(); + subManager_ = std::make_shared(); static_cast(context_)->SetSubscribeManager(subManager_); } else { context_ = new (std::nothrow) MultiVerSyncTaskContext; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h index f6d93ff99..5132c0dd6 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h @@ -60,7 +60,7 @@ private: std::string remoteDeviceId_; SyncTaskContext *context_; std::function onRemoteDataChanged_; - std::shared_ptr subManager_; + std::shared_ptr subManager_; }; } // namespace DistributedDB -- Gitee From 8ab7f46892055f643636e5b3966afd0acf35fa0c Mon Sep 17 00:00:00 2001 From: lianhuix Date: Sat, 25 Dec 2021 16:27:18 +0800 Subject: [PATCH 11/53] Add transaction for create distributed table. Save schema to meta table Signed-off-by: lianhuix --- .../relational/sqlite_relational_store.cpp | 29 ++++++++++++++++--- .../relational/sqlite_relational_store.h | 1 + 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index b3f7e305a..45d88eb4f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -114,6 +114,18 @@ int SQLiteRelationalStore::GetSchemaFromMeta() return E_OK; } +int SQLiteRelationalStore::SaveSchemaToMeta() +{ + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + DBCommon::StringToVector(properties_.GetSchema().ToSchemaString(), schemaVal); + int errCode = storageEngine_->PutMetaData(schemaKey, schemaVal); + if (errCode != E_OK) { + LOGE("Save relational schema to meta table failed. %d", errCode); + } + return errCode; +} + int SQLiteRelationalStore::SaveLogTableVersionToMeta() { // TODO: save log table version into meta data @@ -310,17 +322,26 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) return errCode; } + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + TableInfo table; errCode = handle->CreateDistributedTable(tableName, table); if (errCode != E_OK) { LOGE("create distributed table failed. %d", errCode); - } else { - schema.AddRelationalTable(table); - properties_.SetSchema(schema); + (void)handle->Rollback(); + ReleaseHandle(handle); + return errCode; } + schema.AddRelationalTable(table); + properties_.SetSchema(schema); + (void)handle->Commit(); ReleaseHandle(handle); - return errCode; + return SaveSchemaToMeta(); } } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index 1d069659f..ecb9b1f6d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -59,6 +59,7 @@ private: void DecreaseConnectionCounter(); int GetSchemaFromMeta(); + int SaveSchemaToMeta(); int SaveLogTableVersionToMeta(); -- Gitee From 53dfeab00cc93bd6bb6e53f45dc15b4128672a93 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Sat, 25 Dec 2021 17:54:10 +0800 Subject: [PATCH 12/53] Add journal and synchronous mode check Signed-off-by: lianhuix --- .../relational/sqlite_relational_store.cpp | 21 ++++++++ .../relational/sqlite_relational_store.h | 2 +- ...single_ver_relational_storage_executor.cpp | 19 +++++++ ...e_single_ver_relational_storage_executor.h | 2 + .../storage/src/sqlite/sqlite_utils.cpp | 50 +++++++++++++++++++ .../storage/src/sqlite/sqlite_utils.h | 4 ++ 6 files changed, 97 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 45d88eb4f..d882fe91f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -87,6 +87,22 @@ void SQLiteRelationalStore::ReleaseResources() } } +int SQLiteRelationalStore::CheckDBMode() +{ + int errCode = E_OK; + auto *handle = GetHandle(false, errCode); + if (handle == nullptr) { + return errCode; + } + errCode = handle->CheckDBModeForRelational(); + if (errCode != E_OK) { + LOGE("check relational DB mode failed. %d", errCode); + } + + ReleaseHandle(handle); + return errCode; +} + int SQLiteRelationalStore::GetSchemaFromMeta() { const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); @@ -182,6 +198,11 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } + errCode = CheckDBMode(); + if (errCode != E_OK) { + break; + } + properties_ = properties; errCode = GetSchemaFromMeta(); if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index ecb9b1f6d..dd0b50360 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -57,7 +57,7 @@ private: // 1 store 1 connection void DecreaseConnectionCounter(); - + int CheckDBMode(); int GetSchemaFromMeta(); int SaveSchemaToMeta(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 57e6a7181..f7ce29b93 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -827,5 +827,24 @@ int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataByQuery(std::vector Date: Sat, 25 Dec 2021 19:10:11 +0800 Subject: [PATCH 13/53] Open relational db without set journal mode Signed-off-by: lianhuix --- .../sqlite_single_relational_storage_engine.cpp | 2 +- .../sqlite_single_ver_relational_storage_executor.cpp | 2 +- .../distributeddb/storage/src/sqlite/sqlite_utils.cpp | 11 +++++++---- .../distributeddb/storage/src/sqlite/sqlite_utils.h | 4 ++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index 6c9a5390f..6ef4b916e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -52,7 +52,7 @@ int SQLiteSingleRelationalStorageEngine::RegisterFunction(sqlite3 *db) const int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, StorageExecutor *&handle) { sqlite3 *db = nullptr; - int errCode = SQLiteUtils::OpenDatabase(option_, db); + int errCode = SQLiteUtils::OpenDatabase(option_, db, false); if (errCode != E_OK) { return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index f7ce29b93..3c029f027 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -833,7 +833,7 @@ int SQLiteSingleVerRelationalStorageExecutor::CheckDBModeForRelational() std::string journalMode; int errCode = SQLiteUtils::GetJournalMode(dbHandle_, journalMode); if (errCode != E_OK || journalMode != "wal") { - LOGE("Not support journal mode %s for relational db, expect wal mode, %d", journalMode, errCode); + LOGE("Not support journal mode %s for relational db, expect wal mode, %d", journalMode.c_str(), errCode); return -E_NOT_SUPPORT; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index c9cc7d7c8..8d2b3ad1a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -102,7 +102,7 @@ std::string GetTriggerModeString(TriggerModeEnum mode) } } -int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp) +int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp, bool setWal) { uint64_t flag = SQLITE_OPEN_URI | SQLITE_OPEN_READWRITE; if (properties.createIfNecessary) { @@ -114,7 +114,10 @@ int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&db return -E_INVALID_ARGS; } std::string defaultAttachCipher = DEFAULT_ATTACH_CIPHER + cipherName + ";"; - std::vector sqls {WAL_MODE_SQL, defaultAttachCipher, DEFAULT_ATTACH_KDF_ITER}; + std::vector sqls {defaultAttachCipher, DEFAULT_ATTACH_KDF_ITER}; + if (setWal) { + sqls.push_back(WAL_MODE_SQL); + } std::string fileUrl = DBConstant::SQLITE_URL_PRE + properties.uri; int errCode = sqlite3_open_v2(fileUrl.c_str(), &dbTemp, flag, nullptr); @@ -139,7 +142,7 @@ END: return errCode; } -int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db) +int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db, bool setWal) { { // Only for register the sqlite3 log callback @@ -150,7 +153,7 @@ int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db) } } sqlite3 *dbTemp = nullptr; - int errCode = CreateDataBase(properties, dbTemp); + int errCode = CreateDataBase(properties, dbTemp, setWal); if (errCode != E_OK) { goto END; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 864169a63..8452f705a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -69,7 +69,7 @@ struct OpenDbProperties { class SQLiteUtils { public: // Initialize the SQLiteUtils with the given properties. - static int OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db); + static int OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db, bool setWal = true); // Check the statement and prepare the new if statement is null static int GetStatement(sqlite3 *db, const std::string &sql, sqlite3_stmt *&statement); @@ -182,7 +182,7 @@ public: private: - static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp); + static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp, bool setWal); static int SetBusyTimeout(sqlite3 *db, int timeout); -- Gitee From 411622f573d588d83a777308f9ae0de9a779ad74 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 27 Dec 2021 10:05:40 +0800 Subject: [PATCH 14/53] Split DBProperties into KV & Relational Signed-off-by: lianhuix --- .../libs/distributeddb/BUILD.gn | 2 + .../relational/relational_store_manager.h | 2 +- .../relational/relational_store_instance.h | 2 +- .../storage/include/db_properties.h | 61 +++ .../storage/include/kvdb_properties.h | 59 +-- .../storage/include/relationaldb_properties.h | 44 ++ .../storage/src/db_properties.cpp | 72 ++++ .../storage/src/irelational_store.h | 2 +- .../storage/src/kvdb_properties.cpp | 75 ---- .../storage/src/relationaldb_properties.cpp | 39 ++ .../src/single_ver_subscribe_manager.cpp | 394 ------------------ .../syncer/src/single_ver_subscribe_manager.h | 130 ------ .../libs/distributeddb/test/BUILD.gn | 2 + .../unittest/common/syncer/virtual_device.cpp | 305 -------------- .../unittest/common/syncer/virtual_device.h | 67 --- 15 files changed, 224 insertions(+), 1032 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h create mode 100644 services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp delete mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp delete mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h delete mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp delete mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index 05ae98eac..924097bb6 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -127,6 +127,7 @@ ohos_shared_library("distributeddb") { "interfaces/src/relational/relational_store_sqlite_ext.cpp", "interfaces/src/relational/runtime_config.cpp", "storage/src/data_transformer.cpp", + "storage/src/db_properties.cpp", "storage/src/default_factory.cpp", "storage/src/generic_kvdb.cpp", "storage/src/generic_kvdb_connection.cpp", @@ -160,6 +161,7 @@ ohos_shared_library("distributeddb") { "storage/src/relational_store_connection.cpp", "storage/src/relational_store_instance.cpp", "storage/src/relational_sync_able_storage.cpp", + "storage/src/relationaldb_properties.cpp", "storage/src/result_entries_window.cpp", "storage/src/single_ver_natural_store_commit_notify_data.cpp", "storage/src/sqlite/query_object.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 459196bdf..71fc2e44c 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -22,7 +22,7 @@ #include "auto_launch_export.h" #include "relational_store_delegate.h" #include "irelational_store.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" #include "types.h" namespace DistributedDB { diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h index 1bed16fe0..5c24c1f2c 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h @@ -20,7 +20,7 @@ #include #include "irelational_store.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" namespace DistributedDB { class RelationalStoreInstance final { diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h new file mode 100644 index 000000000..1bb6f0baf --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DB_PROPERTIES_H +#define DB_PROPERTIES_H + +#include +#include +#include + +namespace DistributedDB { +class DBProperties { +public: + // Get the string property according the name + std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; + + // Set the string property for the name + void SetStringProp(const std::string &name, const std::string &value); + + // Get the bool property according the name + bool GetBoolProp(const std::string &name, bool defaultValue) const; + + // Set the bool property for the name + void SetBoolProp(const std::string &name, bool value); + + // Get the bool property according the name + int GetIntProp(const std::string &name, int defaultValue) const; + + // Set the integer property for the name + void SetIntProp(const std::string &name, int value); + + static const std::string CREATE_IF_NECESSARY; + static const std::string DATABASE_TYPE; + static const std::string DATA_DIR; + static const std::string USER_ID; + static const std::string APP_ID; + static const std::string STORE_ID; + static const std::string IDENTIFIER_DATA; + static const std::string IDENTIFIER_DIR; + +protected: + DBProperties() = default; + virtual ~DBProperties() = default; + + std::map stringProperties_; + std::map boolProperties_; + std::map intProperties_; +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h index 7250af7a5..c985b0d83 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h @@ -20,48 +20,10 @@ #include #include +#include "db_properties.h" #include "schema_object.h" -#include "relational_schema_object.h" namespace DistributedDB { -class DBProperties { -public: - DBProperties() = default; - virtual ~DBProperties() = default; - - // Get the string property according the name - std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; - - // Set the string property for the name - void SetStringProp(const std::string &name, const std::string &value); - - // Get the bool property according the name - bool GetBoolProp(const std::string &name, bool defaultValue) const; - - // Set the bool property for the name - void SetBoolProp(const std::string &name, bool value); - - // Get the bool property according the name - int GetIntProp(const std::string &name, int defaultValue) const; - - // Set the integer property for the name - void SetIntProp(const std::string &name, int value); - - static const std::string CREATE_IF_NECESSARY; - static const std::string DATABASE_TYPE; - static const std::string DATA_DIR; - static const std::string USER_ID; - static const std::string APP_ID; - static const std::string STORE_ID; - static const std::string IDENTIFIER_DATA; - static const std::string IDENTIFIER_DIR; - -protected: - std::map stringProperties_; - std::map boolProperties_; - std::map intProperties_; -}; - class KvDBProperties final : public DBProperties { public: KvDBProperties(); @@ -117,25 +79,6 @@ private: CipherPassword password_; SchemaObject schema_; }; - -// TODO: move to its own file, or rename this file -class RelationalDBProperties final : public DBProperties { -public: - RelationalDBProperties(); - ~RelationalDBProperties() override; - - // is schema exist - bool IsSchemaExist() const; - - // set schema - void SetSchema(const RelationalSchemaObject &schema); - - // get schema - RelationalSchemaObject GetSchema() const; - -private: - RelationalSchemaObject schema_; -}; } // namespace DistributedDB #endif // KV_DB_PROPERTIES_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h new file mode 100644 index 000000000..4877e7220 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONALDB_PROPERTIES_H +#define RELATIONALDB_PROPERTIES_H + +#include +#include +#include + +#include "db_properties.h" +#include "relational_schema_object.h" + +namespace DistributedDB { +class RelationalDBProperties final : public DBProperties { +public: + RelationalDBProperties(); + ~RelationalDBProperties() override; + + // is schema exist + bool IsSchemaExist() const; + + // set schema + void SetSchema(const RelationalSchemaObject &schema); + + // get schema + RelationalSchemaObject GetSchema() const; + +private: + RelationalSchemaObject schema_; +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp new file mode 100644 index 000000000..92ecca798 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "db_properties.h" + +namespace DistributedDB { +const std::string DBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; +const std::string DBProperties::DATABASE_TYPE = "databaseType"; +const std::string DBProperties::DATA_DIR = "dataDir"; +const std::string DBProperties::USER_ID = "userId"; +const std::string DBProperties::APP_ID = "appId"; +const std::string DBProperties::STORE_ID = "storeId"; +const std::string DBProperties::IDENTIFIER_DATA = "identifier"; +const std::string DBProperties::IDENTIFIER_DIR = "identifierDir"; + +std::string DBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const +{ + auto iter = stringProperties_.find(name); + if (iter != stringProperties_.end()) { + return iter->second; + } else { + return defaultValue; + } +} + +void DBProperties::SetStringProp(const std::string &name, const std::string &value) +{ + stringProperties_[name] = value; +} + +bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const +{ + auto iter = boolProperties_.find(name); + if (iter != boolProperties_.end()) { + return iter->second; + } else { + return defaultValue; + } +} + +void DBProperties::SetBoolProp(const std::string &name, bool value) +{ + boolProperties_[name] = value; +} + +int DBProperties::GetIntProp(const std::string &name, int defaultValue) const +{ + auto iter = intProperties_.find(name); + if (iter != intProperties_.end()) { + return iter->second; + } else { + return defaultValue; + } +} + +void DBProperties::SetIntProp(const std::string &name, int value) +{ + intProperties_[name] = value; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h index 5ff966e9e..7695b7877 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h @@ -20,7 +20,7 @@ #include #include "ref_object.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" #include "relational_store_connection.h" namespace DistributedDB { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp index 75e98ded8..c10569d82 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp @@ -18,15 +18,6 @@ #include "db_constant.h" namespace DistributedDB { -const std::string DBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; -const std::string DBProperties::DATABASE_TYPE = "databaseType"; -const std::string DBProperties::DATA_DIR = "dataDir"; -const std::string DBProperties::USER_ID = "userId"; -const std::string DBProperties::APP_ID = "appId"; -const std::string DBProperties::STORE_ID = "storeId"; -const std::string DBProperties::IDENTIFIER_DATA = "identifier"; -const std::string DBProperties::IDENTIFIER_DIR = "identifierDir"; - const std::string KvDBProperties::FILE_NAME = "fileName"; const std::string KvDBProperties::MEMORY_MODE = "memoryMode"; const std::string KvDBProperties::ENCRYPTED_MODE = "isEncryptedDb"; @@ -60,51 +51,6 @@ std::string KvDBProperties::GetStoreSubDirectory(int type) } } -std::string DBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const -{ - auto iter = stringProperties_.find(name); - if (iter != stringProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetStringProp(const std::string &name, const std::string &value) -{ - stringProperties_[name] = value; -} - -bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const -{ - auto iter = boolProperties_.find(name); - if (iter != boolProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetBoolProp(const std::string &name, bool value) -{ - boolProperties_[name] = value; -} - -int DBProperties::GetIntProp(const std::string &name, int defaultValue) const -{ - auto iter = intProperties_.find(name); - if (iter != intProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetIntProp(const std::string &name, int value) -{ - intProperties_[name] = value; -} - void KvDBProperties::GetPassword(CipherType &type, CipherPassword &password) const { type = cipherType_; @@ -146,25 +92,4 @@ const SchemaObject &KvDBProperties::GetSchemaConstRef() const { return schema_; } - -RelationalDBProperties::RelationalDBProperties() -{} - -RelationalDBProperties::~RelationalDBProperties() -{} - -bool RelationalDBProperties::IsSchemaExist() const -{ - return schema_.IsSchemaValid(); -} - -void RelationalDBProperties::SetSchema(const RelationalSchemaObject &schema) -{ - schema_ = schema; -} - -RelationalSchemaObject RelationalDBProperties::GetSchema() const -{ - return schema_; -} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp new file mode 100644 index 000000000..3a87aafb5 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "relationaldb_properties.h" + +namespace DistributedDB { +RelationalDBProperties::RelationalDBProperties() +{} + +RelationalDBProperties::~RelationalDBProperties() +{} + +bool RelationalDBProperties::IsSchemaExist() const +{ + return schema_.IsSchemaValid(); +} + +void RelationalDBProperties::SetSchema(const RelationalSchemaObject &schema) +{ + schema_ = schema; +} + +RelationalSchemaObject RelationalDBProperties::GetSchema() const +{ + return schema_; +} +} diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp deleted file mode 100644 index ea78cbe1b..000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "single_ver_subscribe_manager.h" -#include "db_common.h" -#include "sync_types.h" - -namespace DistributedDB { -void SingleVerSubscribeManager::ClearRemoteSubscribeQuery(const std::string &device) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - ClearSubscribeQuery(device, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::ClearAllRemoteQuery() -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - remoteSubscribedMap_.clear(); - remoteSubscribedTotalMap_.clear(); -} - -void SingleVerSubscribeManager::ClearLocalSubscribeQuery(const std::string &device) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - unFinishedLocalAutoSubMap_.erase(device); - ClearSubscribeQuery(device, localSubscribeMap_, localSubscribeTotalMap_); -} - -int SingleVerSubscribeManager::ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - int errCode = ReserveSubscribeQuery(device, query, remoteSubscribedMap_, remoteSubscribedTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote reserve err=%d", - STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - int errCode = ActiveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote active err=%d", - STR_MASK(device), STR_MASK(queryId), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - int errCode = ReserveSubscribeQuery(device, query, localSubscribeMap_, localSubscribeTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local reserve err=%d", - STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - int errCode = ActiveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local active err=%d", - STR_MASK(device), STR_MASK(queryId), errCode); - if (errCode != E_OK) { - return errCode; - } - if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && - unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { - unFinishedLocalAutoSubMap_[device].erase(queryId); - } - return errCode; -} - -void SingleVerSubscribeManager::DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - DeleteSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); -} - -void SingleVerSubscribeManager::DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - DeleteSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::PutLocalUnFiniedSubQueries(const std::string &device, - std::vector &subscribeQueries) -{ - LOGI("[SingleVerSubscribeManager] put local unfinished subscribe queries, nums=%d", subscribeQueries.size()); - std::unique_lock lockGuard(localSubscribeMapLock_); - if (subscribeQueries.size() == 0) { - unFinishedLocalAutoSubMap_.erase(device); - return; - } - unFinishedLocalAutoSubMap_[device].clear(); - auto iter = unFinishedLocalAutoSubMap_.find(device); - for (const auto &query : subscribeQueries) { - iter->second.insert(query.GetIdentify()); - } -} - -void SingleVerSubscribeManager::GetAllUnFinishSubQueries( - std::map> &allSyncQueries) const -{ - std::shared_lock lock(localSubscribeMapLock_); - for (auto &item : unFinishedLocalAutoSubMap_) { - if (item.second.size() == 0) { - continue; - } - allSyncQueries[item.first] = {}; - auto iter = allSyncQueries.find(item.first); - for (const auto &queryId : item.second) { - auto iterTmp = localSubscribeTotalMap_.find(queryId); - if (iterTmp == localSubscribeTotalMap_.end()) { - LOGI("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryId)); - continue; - } - iter->second.push_back(iterTmp->second.first); - } - } -} - -void SingleVerSubscribeManager::RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - RemoveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - RemoveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); - if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && - unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { - unFinishedLocalAutoSubMap_[device].erase(queryId); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s delete from UnFinishedMap", - STR_MASK(device), STR_MASK(queryId)); - if (unFinishedLocalAutoSubMap_[device].size() == 0) { - LOGI("[SingleVerSubscribeManager] dev=%s delete from unFinish map", STR_MASK(device)); - unFinishedLocalAutoSubMap_.erase(device); - } - } -} - -void SingleVerSubscribeManager::GetLocalSubscribeQueries(const std::string &device, - std::vector &subscribeQueries) const -{ - std::shared_lock lock(localSubscribeMapLock_); - GetSubscribeQueries(device, localSubscribeMap_, localSubscribeTotalMap_, subscribeQueries); -} - -void SingleVerSubscribeManager::GetRemoteSubscribeQueries(const std::string &device, - std::vector &subscribeQueries) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - GetSubscribeQueries(device, remoteSubscribedMap_, remoteSubscribedTotalMap_, subscribeQueries); -} - -bool SingleVerSubscribeManager::IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - auto iter = remoteSubscribedMap_.find(device); - if (iter == remoteSubscribedMap_.end()) { - LOGD("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); - return false; - } - std::string queryId = query.GetIdentify(); - auto subIter = iter->second.find(queryId); - if (subIter == iter->second.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryId)); - return false; - } - return true; -} - -void SingleVerSubscribeManager::GetRemoteSubscribeQueryIds(const std::string &device, - std::vector &subscribeQueryIds) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - auto iter = remoteSubscribedMap_.find(device); - if (iter == remoteSubscribedMap_.end()) { - LOGI("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : iter->second) { - if (remoteSubscribedTotalMap_.find(queryInfo.first) == remoteSubscribedTotalMap_.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryInfo.first)); - continue; - } - subscribeQueryIds.push_back(queryInfo.first); - } -} - -int SingleVerSubscribeManager::LocalSubscribeLimitCheck(const std::vector &devices, - QuerySyncObject &query) const -{ - std::shared_lock lock(localSubscribeMapLock_); - int devNum = localSubscribeMap_.size(); - for (const auto &device : devices) { - if (localSubscribeMap_.find(device) != localSubscribeMap_.end()) { - continue; - } - devNum++; - if (devNum > MAX_DEVICES_NUM) { - LOGE("[SingleVerSubscribeManager] local subscribe devices is over limit"); - return -E_MAX_LIMITS; - } - } - std::string queryId = query.GetIdentify(); - auto allIter = localSubscribeTotalMap_.find(queryId); - if (allIter == localSubscribeTotalMap_.end() && localSubscribeTotalMap_.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { - LOGE("[SingleVerSubscribeManager] all local subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - return E_OK; -} - -void SingleVerSubscribeManager::ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGI("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : subscribeMap[device]) { - if (subscribedTotalMap.find(queryInfo.first) != subscribedTotalMap.end()) { - if (subscribedTotalMap[queryInfo.first].second > 0) { - subscribedTotalMap[queryInfo.first].second--; - } - if (subscribedTotalMap[queryInfo.first].second == 0) { - LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryInfo.first)); - subscribedTotalMap.erase(queryInfo.first); - } - } - } - subscribeMap.erase(device); - LOGI("[SingleVerSubscribeManager] clear dev=%s` remote subscribe queies finished", STR_MASK(device)); -} - -int SingleVerSubscribeManager::ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - std::string queryId = query.GetIdentify(); - auto iter = subscribeMap.find(device); - auto allIter = subscribedTotalMap.find(queryId); - // limit check - if (allIter == subscribedTotalMap.end() && subscribedTotalMap.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { - LOGE("[SingleVerSubscribeManager] all subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - if (iter == subscribeMap.end() && subscribeMap.size() >= MAX_DEVICES_NUM) { - LOGE("[SingleVerSubscribeManager] subscribe devices is over limit"); - return -E_MAX_LIMITS; - } - if (iter != subscribeMap.end() && iter->second.find(queryId) == iter->second.end() && - iter->second.size() >= MAX_SUBSCRIBE_NUM_PER_DEV) { - LOGE("[SingleVerSubscribeManager] subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - if (iter != subscribeMap.end() && iter->second.find(queryId) != iter->second.end() && - iter->second[queryId] == SubscribeStatus::ACTIVE) { - LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s already active in map", - STR_MASK(device), STR_MASK(queryId)); - return E_OK; - } - - if (iter == subscribeMap.end()) { - subscribeMap[device] = std::map {}; - } - bool isNeedInc = false; - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - subscribeMap[device][queryId] = SubscribeStatus::NOT_ACTIVE; - isNeedInc = true; - } - if (allIter == subscribedTotalMap.end()) { - subscribedTotalMap[queryId] = {query, 1}; - } else if (isNeedInc) { - subscribedTotalMap[queryId].second++; - } - return E_OK; -} - -int SingleVerSubscribeManager::ActiveSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribedTotalMap.find(queryId) == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); - return -E_INTERNAL_ERROR; - } - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s in localSubscribeMap", STR_MASK(device)); - return -E_INTERNAL_ERROR; - } - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); - return -E_INTERNAL_ERROR; - } - subscribeMap[device][queryId] = SubscribeStatus::ACTIVE; - return E_OK; -} - -void SingleVerSubscribeManager::DeleteSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s in map", STR_MASK(device)); - return; - } - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); - return; - } - SubscribeStatus queryStatus = subscribeMap[device][queryId]; - // not permit to delete the query when something wrong this time,because it is subscribed successfully last time - if (queryStatus == SubscribeStatus::ACTIVE) { - LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s is active, no need to del", - STR_MASK(device), STR_MASK(queryId)); - return; - } - subscribeMap[device].erase(queryId); - auto iter = subscribedTotalMap.find(queryId); - if (iter == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); - return; - } - iter->second.second--; - if (iter->second.second <= 0) { - LOGI("[SingleVerSubscribeManager] del queryId=%s from SubscribeTotalMap", STR_MASK(queryId)); - subscribedTotalMap.erase(queryId); - } - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribeMap success", - STR_MASK(device), STR_MASK(queryId)); -} - -void SingleVerSubscribeManager::RemoveSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - auto iter = subscribeMap.find(device); - if (iter == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); - return; - } - if (iter->second.find(queryId) == subscribeMap[device].end()) { - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s not in SubscribedMap", STR_MASK(device), STR_MASK(queryId)); - return; - } - iter->second.erase(queryId); - auto allIter = subscribedTotalMap.find(queryId); - if (allIter == subscribedTotalMap.end()) { - LOGI("[SingleVerSubscribeManager] queryId=%s not in TotalMap", STR_MASK(queryId)); - return; - } - allIter->second.second--; - if (allIter->second.second <= 0) { - subscribedTotalMap.erase(queryId); - LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryId)); - } - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribedMap success", - STR_MASK(device), STR_MASK(queryId)); -} - -void SingleVerSubscribeManager::GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, - const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const -{ - auto iter = subscribeMap.find(device); - if (iter == subscribeMap.end()) { - LOGD("[SingleVerSubscribeManager] dev=%s not in localSubscribeMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : iter->second) { - auto iterTmp = subscribedTotalMap.find(queryInfo.first); - if (iterTmp == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryInfo.first)); - continue; - } - subscribeQueries.push_back(iterTmp->second.first); - } -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h deleted file mode 100644 index c152e0f29..000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SINGLE_VER_SUBSCRIBE_MANAGER_H -#define SINGLE_VER_SUBSCRIBE_MANAGER_H - -#include -#include -#include "query_sync_object.h" - -namespace DistributedDB { -enum class SubscribeStatus { - NOT_ACTIVE = 0, - ACTIVE = 1, -}; - -using SubscribeMap = std::map>; -using SubscribedTotalMap = std::map>; - -class SingleVerSubscribeManager { -public: - SingleVerSubscribeManager() = default; - ~SingleVerSubscribeManager() {}; - - DISABLE_COPY_ASSIGN_MOVE(SingleVerSubscribeManager); - - // clear remoteSubscribeMap_[device] list when remote db is closed or dev offline. - void ClearRemoteSubscribeQuery(const std::string &device); - - // clear localSubscribeMap_[device] list when device is offline. - void ClearLocalSubscribeQuery(const std::string &device); - - void ClearAllRemoteQuery(); - - // add query when receive subscribe command - int ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // active query to ACTIVE when send ack ok - int ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // reserve query when user call SubscribeRemoteQuery, status set to NOT_ACTIVE - int ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // active query to ACTIVE when receive ack ok - int ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // delete local subscribe query when recv wrong errCode, only not_active status allowed to del - void DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // delete remote subscribe query when send msg failed, only not_active status allowed to del - void DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // put subscribe queries into unfinished map when remote db online - void PutLocalUnFiniedSubQueries(const std::string &device, std::vector &subscribeQueries); - - // get all device unFinished subscribe queries which triggered by auto subscribe and need retry subscribe - void GetAllUnFinishSubQueries(std::map> &allSyncQueries) const; - - // remove query when receive unsubscribe command - void RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // remove query when user call UnSubscribeRemoteQuery - void RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // get device active subscribeQueries from localSubscribeMap_ - void GetLocalSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; - - // get device remote queryId from remoteSubscribedMap_ while data change - void GetRemoteSubscribeQueryIds(const std::string &device, std::vector &subscribeQueryIds) const; - // get device remote subscribeQueries from remoteSubscribedMap_ while data change - void GetRemoteSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; - - bool IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const; - - int LocalSubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const; -private: - void ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - int ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - int ActiveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void DeleteSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void RemoveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, - const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const; - - mutable std::shared_mutex localSubscribeMapLock_; - // subscribe sponsor, key: device, value: pair map - // status 0: active, 1: not active - SubscribeMap localSubscribeMap_; - - // used retry subscribe in db open scene, key: device value: set - std::map> unFinishedLocalAutoSubMap_; - - // subscribe sponsor total query info, key:queryId, value: - // while use_num is 0, delete item from the map - SubscribedTotalMap localSubscribeTotalMap_; - - mutable std::shared_mutex remoteSubscribedMapLock_; - // subscribed, key: device, value: pair map - // status 0: active, 1: not active - SubscribeMap remoteSubscribedMap_; - - // subscribed total query info, key:queryId, value: - // while use_num is 0, delete item from the map - SubscribedTotalMap remoteSubscribedTotalMap_; -}; -} // namespace DistributedDB - -#endif // SINGLE_VER_SUBSCRIBE_MANAGER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index c181d617b..22581c572 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -130,6 +130,7 @@ ohos_source_set("src_file") { "../interfaces/src/relational/relational_store_sqlite_ext.cpp", "../interfaces/src/relational/runtime_config.cpp", "../storage/src/data_transformer.cpp", + "../storage/src/db_properties.cpp", "../storage/src/default_factory.cpp", "../storage/src/generic_kvdb.cpp", "../storage/src/generic_kvdb_connection.cpp", @@ -163,6 +164,7 @@ ohos_source_set("src_file") { "../storage/src/relational_store_connection.cpp", "../storage/src/relational_store_instance.cpp", "../storage/src/relational_sync_able_storage.cpp", + "../storage/src/relationaldb_properties.cpp", "../storage/src/result_entries_window.cpp", "../storage/src/single_ver_natural_store_commit_notify_data.cpp", "../storage/src/sqlite/query_object.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp deleted file mode 100644 index 25749ee91..000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "virtual_device.h" - -#include -#include - -#include "device_manager.h" -#include "log_print.h" -#include "multi_ver_sync_state_machine.h" -#include "subscribe_manager.h" -#include "single_ver_sync_state_machine.h" -#include "sync_types.h" -#include "virtual_communicator.h" -#include "virtual_communicator_aggregator.h" -#include "virtual_multi_ver_sync_db_interface.h" -#include "virtual_single_ver_sync_db_Interface.h" - -namespace DistributedDB { -VirtualDevice::VirtualDevice(const std::string &deviceId) - : communicateHandle_(nullptr), - communicatorAggregator_(nullptr), - storage_(nullptr), - metadata_(nullptr), - deviceId_(deviceId), - remoteDeviceId_("real_device"), - context_(nullptr) -{ -} - -VirtualDevice::~VirtualDevice() -{ - std::mutex cvMutex; - std::condition_variable cv; - bool finished = false; - Offline(); - - if (communicateHandle_ != nullptr) { - communicateHandle_->RegOnMessageCallback(nullptr, nullptr); - communicatorAggregator_->ReleaseCommunicator(communicateHandle_); - communicateHandle_ = nullptr; - } - communicatorAggregator_ = nullptr; - - if (context_ != nullptr) { - IKvDBSyncInterface *storage = storage_; - context_->OnLastRef([storage, &cv, &cvMutex, &finished]() { - if (storage != nullptr) { - delete storage; - } - { - std::lock_guard lock(cvMutex); - finished = true; - } - cv.notify_one(); - }); - RefObject::KillAndDecObjRef(context_); - std::unique_lock lock(cvMutex); - cv.wait(lock, [&finished] {return finished;}); - } else { - delete storage_; - } - context_ = nullptr; - metadata_ = nullptr; - storage_ = nullptr; - subManager_ = nullptr; -} - -int VirtualDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface) -{ - if ((communicatorAggregator == nullptr) || (syncInterface == nullptr)) { - return -E_INVALID_ARGS; - } - - communicatorAggregator_ = communicatorAggregator; - int errCode = E_OK; - communicateHandle_ = communicatorAggregator_->AllocCommunicator(deviceId_, errCode); - if (communicateHandle_ == nullptr) { - return errCode; - } - - storage_ = syncInterface; - metadata_ = std::make_shared(); - if (metadata_->Initialize(storage_) != E_OK) { - LOGE("metadata_ init failed"); - return -E_NOT_SUPPORT; - } - if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_SVD) { - context_ = new (std::nothrow) SingleVerSyncTaskContext; - subManager_ = std::make_shared(); - static_cast(context_)->SetSubscribeManager(subManager_); - } else { - context_ = new (std::nothrow) MultiVerSyncTaskContext; - } - if (context_ == nullptr) { - return -E_OUT_OF_MEMORY; - } - communicateHandle_->RegOnMessageCallback(std::bind(&VirtualDevice::MessageCallback, this, - std::placeholders::_1, std::placeholders::_2), []() {}); - context_->Initialize(remoteDeviceId_, storage_, metadata_, communicateHandle_); - context_->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); - context_->RegOnSyncTask(std::bind(&VirtualDevice::StartResponseTask, this)); - return E_OK; -} - -void VirtualDevice::SetDeviceId(const std::string &deviceId) -{ - deviceId_ = deviceId; -} - -std::string VirtualDevice::GetDeviceId() const -{ - return deviceId_; -} - -int VirtualDevice::GetData(const Key &key, VirtualDataItem &item) -{ - VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); - return syncAble->GetSyncData(key, item); -} - -int VirtualDevice::GetData(const Key &key, Value &value) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->GetData(key, value); -} - -int VirtualDevice::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) -{ - VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); - LOGI("dev %s put data time %llu", deviceId_.c_str(), time); - return syncAble->PutData(key, value, time, flag); -} - -int VirtualDevice::PutData(const Key &key, const Value &value) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->PutData(key, value); -} - -int VirtualDevice::DeleteData(const Key &key) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->DeleteData(key); -} - -int VirtualDevice::StartTransaction() -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->StartTransaction(); -} - -int VirtualDevice::Commit() -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->Commit(); -} - -int VirtualDevice::MessageCallback(const std::string &deviceId, Message *inMsg) -{ - if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { - if (onRemoteDataChanged_) { - onRemoteDataChanged_(deviceId); - delete inMsg; - inMsg = nullptr; - return E_OK; - } - delete inMsg; - inMsg = nullptr; - return -E_INVALID_ARGS; - } - - LOGD("[VirtualDevice] onMessag, src %s", deviceId.c_str()); - RefObject::IncObjRef(context_); - RefObject::IncObjRef(communicateHandle_); - SyncTaskContext *context = context_; - ICommunicator *communicateHandle = communicateHandle_; - std::thread thread([context, communicateHandle, inMsg]() { - int errCode = context->ReceiveMessageCallback(inMsg); - if (errCode != -E_NOT_NEED_DELETE_MSG) { - delete inMsg; - } - RefObject::DecObjRef(context); - RefObject::DecObjRef(communicateHandle); - }); - thread.detach(); - return E_OK; -} - -void VirtualDevice::OnRemoteDataChanged(const std::function &callback) -{ - onRemoteDataChanged_ = callback; -} - -void VirtualDevice::Online() -{ - static_cast(communicateHandle_)->Enable(); - communicatorAggregator_->OnlineDevice(deviceId_); -} - -void VirtualDevice::Offline() -{ - static_cast(communicateHandle_)->Disable(); - communicatorAggregator_->OfflineDevice(deviceId_); -} - -int VirtualDevice::StartResponseTask() -{ - LOGD("[VirtualDevice] StartResponseTask"); - RefObject::AutoLock lockGuard(context_); - int status = context_->GetTaskExecStatus(); - if ((status == SyncTaskContext::RUNNING) || context_->IsKilled()) { - LOGD("[VirtualDevice] StartResponseTask status:%d", status); - return -E_NOT_SUPPORT; - } - if (context_->IsTargetQueueEmpty()) { - LOGD("[VirtualDevice] StartResponseTask IsTargetQueueEmpty is empty"); - return E_OK; - } - context_->SetTaskExecStatus(ISyncTaskContext::RUNNING); - context_->MoveToNextTarget(); - LOGI("[VirtualDevice] machine StartSync"); - context_->UnlockObj(); - int errCode = context_->StartStateMachine(); - context_->LockObj(); - if (errCode != E_OK) { - LOGE("[VirtualDevice] machine StartSync failed"); - context_->SetOperationStatus(SyncOperation::OP_FAILED); - } - return errCode; -} - -TimeOffset VirtualDevice::GetLocalTimeOffset() const -{ - return metadata_->GetLocalTimeOffset(); -} - -void VirtualDevice::SetSaveDataDelayTime(uint64_t milliDelayTime) -{ - VirtualSingleVerSyncDBInterface *syncInterface = static_cast(storage_); - syncInterface->SetSaveDataDelayTime(milliDelayTime); -} - -int VirtualDevice::Sync(SyncMode mode, bool wait) -{ - auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} - -int VirtualDevice::Subscribe(QuerySyncObject query, bool wait, int id) -{ - auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, SUBSCRIBE_QUERY, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - operation->SetQuery(query); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} - -int VirtualDevice::UnSubscribe(QuerySyncObject query, bool wait, int id) -{ - auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, UNSUBSCRIBE_QUERY, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - operation->SetQuery(query); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h deleted file mode 100644 index 5132c0dd6..000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VIRTUAL_DEVICE_H -#define VIRTUAL_DEVICE_H - -#include "icommunicator.h" - -#include "ikvdb_sync_interface.h" -#include "single_ver_sync_task_context.h" -#include "virtual_single_ver_sync_db_Interface.h" - -namespace DistributedDB { -class VirtualCommunicatorAggregator; - -class VirtualDevice { -public: - explicit VirtualDevice(const std::string &deviceId); - ~VirtualDevice(); - - int Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface); - void SetDeviceId(const std::string &deviceId); - std::string GetDeviceId() const; - int GetData(const Key &key, VirtualDataItem &item); - int GetData(const Key &key, Value &value); - int PutData(const Key &key, const Value &value, const TimeStamp &time, int flag); - int PutData(const Key &key, const Value &value); - int DeleteData(const Key &key); - int StartTransaction(); - int Commit(); - int MessageCallback(const std::string &deviceId, Message *inMsg); - void OnRemoteDataChanged(const std::function &callback); - void Online(); - void Offline(); - int StartResponseTask(); - TimeOffset GetLocalTimeOffset() const; - void SetSaveDataDelayTime(uint64_t milliDelayTime); - int Sync(SyncMode mode, bool wait); - int Subscribe(QuerySyncObject query, bool wait, int id); - int UnSubscribe(QuerySyncObject query, bool wait, int id); - -private: - ICommunicator *communicateHandle_; - VirtualCommunicatorAggregator *communicatorAggregator_; - IKvDBSyncInterface *storage_; - std::shared_ptr metadata_; - std::string deviceId_; - std::string remoteDeviceId_; - SyncTaskContext *context_; - std::function onRemoteDataChanged_; - std::shared_ptr subManager_; -}; -} // namespace DistributedDB - -#endif // VIRTUAL_DEVICE_H -- Gitee From 31df7040af8f39bda3b3f0f1b9856b416468edc4 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 27 Dec 2021 10:11:22 +0800 Subject: [PATCH 15/53] Remove TODOs Signed-off-by: lianhuix --- .../relational/relational_schema_object.cpp | 3 +-- .../relational/relational_store_manager.h | 4 ++-- .../include/relational_db_sync_interface.h | 2 -- .../relational/sqlite_relational_store.cpp | 19 +------------------ .../sqlite_relational_store_connection.cpp | 1 - ...single_ver_relational_storage_executor.cpp | 13 +------------ 6 files changed, 5 insertions(+), 37 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp index e5f33dab1..cf784be59 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -296,7 +296,6 @@ JsonObject TableInfo::ToJsonObject() const } for (const auto& it : uniqueDefines_) { jsonField.stringValue = VectorJoin(it, ','); - // TODO: add unique to tableJson } return tableJson; } @@ -646,7 +645,7 @@ int RelationalSchemaObject::ParseCheckTableFieldInfo(const JsonObject &inJsonObj } else if (errCode != -E_NOT_FOUND) { return errCode; } - // TODO: need cid or not? + return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 71fc2e44c..74ccc9423 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -39,9 +39,9 @@ public: DB_API DBStatus OpenStore(const std::string &path, const std::string &storeId, const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate); - DB_API DBStatus CloseStore(RelationalStoreDelegate *store); // TODO: move interface to delegate + DB_API DBStatus CloseStore(RelationalStoreDelegate *store); - DB_API DBStatus DeleteStore(const std::string &path); // TODO: remove interface + DB_API DBStatus DeleteStore(const std::string &path); private: void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h index eff6dbe54..94749dba3 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h @@ -38,8 +38,6 @@ public: virtual int LocalDataChanged(int notifyEvent, std::vector &queryObj) = 0; virtual int SchemaChanged(int notifyEvent) = 0; - - // TODO: create device table for each distributed table. }; } #endif // RELATIONAL_STORE diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index d882fe91f..dad1ed2a3 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -80,7 +80,6 @@ int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &kvDBP void SQLiteRelationalStore::ReleaseResources() { - // TODO: Lock if (sqliteStorageEngine_ != nullptr) { sqliteStorageEngine_->ClearEnginePasswd(); (void)StorageEngineManager::ReleaseStorageEngine(sqliteStorageEngine_); @@ -144,28 +143,12 @@ int SQLiteRelationalStore::SaveSchemaToMeta() int SQLiteRelationalStore::SaveLogTableVersionToMeta() { - // TODO: save log table version into meta data LOGD("save log table version to meta table, key: %s, val: %s", LOG_TABLE_VERSION_KEY, LOG_TABLE_VERSION_1); return E_OK; } int SQLiteRelationalStore::CleanDistributedDeviceTable() { - // TODO: clean the device table which is no longer in schema - std::lock_guard lock(schemaMutex_); - RelationalSchemaObject schema = properties_.GetSchema(); - for (const auto &table : schema.GetTables()) { - std::string tableName = table.first; - LOGD("Get schema %s.", tableName.c_str()); - } - - int errCode = E_OK; - auto *handle = GetHandle(true, errCode); - if (handle == nullptr) { - return errCode; - } - // TODO: Get device table names, and clean - ReleaseHandle(handle); return E_OK; } @@ -193,7 +176,7 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); if (storageEngine_ == nullptr) { - LOGE("[RelationalStore][Open] Create syncable storage failed"); // TODO: + LOGE("[RelationalStore][Open] Create syncable storage failed"); errCode = -E_OUT_OF_MEMORY; break; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index bc86d438c..160a64d91 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -179,7 +179,6 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) syncParam.relationOnComplete = info.onComplete; syncParam.syncQuery = QuerySyncObject(info.query); syncParam.onFinalize = [this]() { DecObjRef(this); }; - // TODO: check if table permit sync or not int errCode = store->Sync(syncParam); if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 3c029f027..892527368 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -46,17 +46,6 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std:: LOGE("[CreateDistributedTable] analysis table schema failed"); return errCode; } - - // TODO: check if compatible upgrade or not - - // TODO: add not permit sync flag if not compatible upgrade - - // TODO: add analysed table to schema. - - // TODO: reset permit sync flag - - // TODO: upgrade device table - // create log table errCode = SQLiteUtils::CreateRelationalLogTable(dbHandle_, tableName); if (errCode != E_OK) { @@ -109,7 +98,7 @@ int SQLiteSingleVerRelationalStorageExecutor::Rollback() int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) { - // TODO: Get table info from schema + int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); if (errCode != E_OK) { LOGE("[CreateDistributedTable] analysis table schema failed"); -- Gitee From ed9d2825fbaa4ea3e28a09fc80285a431f39cc50 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 27 Dec 2021 10:35:54 +0800 Subject: [PATCH 16/53] Change GetTableCount to GetTableRecordCount Signed-off-by: lianhuix --- .../storage/src/sqlite/relational/sqlite_relational_store.cpp | 2 +- .../sqlite/sqlite_single_ver_relational_storage_executor.cpp | 2 +- .../libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp | 2 +- .../libs/distributeddb/storage/src/sqlite/sqlite_utils.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index dad1ed2a3..fe64fc818 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -107,7 +107,7 @@ int SQLiteRelationalStore::GetSchemaFromMeta() const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); Value schemaVal; int errCode = storageEngine_->GetMetaData(schemaKey, schemaVal); - if (errCode != E_OK && errCode != -E_NOT_FOUND ) { + if (errCode != E_OK && errCode != -E_NOT_FOUND) { LOGE("Get relationale schema from meta table failed. %d", errCode); return errCode; } else if (errCode == -E_NOT_FOUND) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 892527368..9fa4e7183 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -30,7 +30,7 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std:: } int historyDataCnt = 0; - int errCode = SQLiteUtils::GetTableCount(dbHandle_, tableName, historyDataCnt); + int errCode = SQLiteUtils::GetTableRecordCount(dbHandle_, tableName, historyDataCnt); if (errCode != E_OK) { LOGE("[CreateDistributedTable] Get the number of table [%s] rows failed. %d", tableName.c_str(), errCode); return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 8d2b3ad1a..906d2952c 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1975,7 +1975,7 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) return E_OK; } -int SQLiteUtils::GetTableCount(sqlite3 *db, const std::string &tableName, int &count) +int SQLiteUtils::GetTableRecordCount(sqlite3 *db, const std::string &tableName, int &count) { if (db == nullptr) { return -E_INVALID_ARGS; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 8452f705a..623335886 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -178,7 +178,7 @@ public: static int ExpandedSql(sqlite3_stmt *stmt, std::string &basicString); - static int GetTableCount(sqlite3 *db, const std::string &tableName, int &count); + static int GetTableRecordCount(sqlite3 *db, const std::string &tableName, int &count); private: -- Gitee From 3efd31fbeb92d20efc0a6724b34d2d30187fb80b Mon Sep 17 00:00:00 2001 From: xuhongkang Date: Mon, 27 Dec 2021 15:42:53 +0800 Subject: [PATCH 17/53] add relational testCase Signed-off-by: xuhongkang --- .../include/distributed_rdb_tools.h | 100 +++++ .../src/distributed_rdb_tools.cpp | 256 +++++++++++++ .../src/distributed_rdb_exception_test.cpp | 353 ++++++++++++++++++ 3 files changed, 709 insertions(+) create mode 100644 services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h create mode 100644 services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp create mode 100644 services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h new file mode 100644 index 000000000..15ce29fd9 --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H +#define DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H +#include +#include + +#include "relational_store_delegate.h" +#include "relational_store_manager.h" +#include "types.h" +#include "distributed_test_sysinfo.h" +#include "distributeddb_data_generator.h" +#include "log_print.h" +#ifdef TESTCASES_USING_GTEST +#define HWTEST_F(test_case_name, test_name, level) TEST_F(test_case_name, test_name) +#endif + +struct RdbParameters { + std::string path; + std::string storeId; + std::string appId; + std::string userId; + RdbParameters(str::string pathStr, std::string storeIdStr, std::string appIdStr, std::string userIdStr) + : pathStr(pathStr), storeId(storeIdStr), appId(appIdStr), userId(userIdStr) + { + } +}; + +const static std::string TAG = "DistributedRdbTestTools"; + +const std::string NORMAL_PATH = "/data/test/"; +const std::string NON_EXISTENT_PATH = "/data/test/nonExistent_rdb/"; +const std::string UNREADABLE_PATH = "/data/test/unreadable_rdb/"; +const std::string UNWRITABLE_PATH = "/data/test/unwritable_rdb/"; + +const std::string NULL_STOREID = ""; +const std::string ILLEGAL_STOREID = "rdb_$%#@~%"; +const std::string MODE_STOREID = "rdb_mode"; +const std::string FULL_STOREID = "rdb_full"; +const std::string SUPER_STOREID = "rdb_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +const std::string NON_EXISTENT_TABLE = "non_table"; +const std::string KEYWORD_START_TABLE = "naturalbase_rdb_a"; + +const std::string NORMAL_TABLE = "NORMAL_RDB"; +const std::string CONSTRAINT_TABLE = "CONSTRAINT_RDB"; + +const static RdbParameters g_rdbParameter1(NORMAL_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter2(NORMAL_PATH, DistributedDBDataGenerator::STORE_ID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter3(NORMAL_PATH, ILLEGAL_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter4(NORMAL_PATH, MODE_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter5(NORMAL_PATH, SUPER_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter6(NON_EXISTENT_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter7(UNREADABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter8(UNWRITABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter9(NORMAL_PATH, FULL_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +class DistributedRdbTestTools final { +public: + DistributedRdbTestTools() {} + ~DistributedRdbTestTools() {} + // Relational Database OpenStore And CreateDistributeTable + static DistributedDB::DBStatus GetOpenStoreStatus(const RelatetionalStoreManager *&manager, RelatetionalStoreDelegate *&delegate, + const RdbParameters ¶m); + static DistributedDB::DBStatus GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, + const std::string &tableName); + static bool CloseStore(const DistributedDB::RelatetionalStoreDelegate *&delegate); +private: +} +#endif // DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H \ No newline at end of file diff --git a/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp new file mode 100644 index 000000000..7ef56037d --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "distributed_rdb_tools.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef USE_SQLITE_SYMBOLS +#include "sqlite3.h" +#else +#include "sqlite3sym.h" +#endif +#include "distributeddb_data_generator.h" +#include "distributed_test_sysinfo.h" +#include "platform_specific.h" +#include "securec.h" + +using namespace std; +using namespace DistributedDB; +using namespace DistributedDBDataGenerator; + +namespace { + std::string gtableNameList[33]; + const char *SQL_CREATE_NORMAL_TABLE = "CREATE TABLE IF NOT EXISTS NORMAL_RDB(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "name VARCHAR(100) NOT NULL DEFAULT \"rdb\");"; + + const char *SQL_CREATE_CONSTRAINT_TABLE = "CREATE TABLE IF NOT EXISTS CONSTRAINT_RDB(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "f_id INT," \ + "age INT NOT NULL UNIQUE," \ + "name VARCHAR(100) NOT NULL DEFAULT \"rdb\"," \ + "address CHAR(50) COLLATE NOCASE," \ + "identity INT NOT NULL, CHECK(identity > 10000) PRIMARY KEY(id, identity)," \ + "FOREGIN key(f_id) references NORMAL_RDB(id));"; + + const char *SQL_CREATE_TRIGGER = "CREATE TRIGGER IF NOT EXISTS insertTrigger " \ + "AFTER INSERT ON CONSTRAINT_RDB " \ + "FOR EACH ROW " \ + "BEGIN " \ + "update NORMAL_RDB set name = \"name_001\" " \ + "END"; + + const char *SQL_INSERT_NORMAL_TABLE = "INSERT INTO NORMAL_RDB (id,name)" \ + "VALUES (1, \'rdb_001\'), (2, \'rdb_002\'), (3, \'rdb_003\'), (4, \'rdb_004\'), (5, \'rdb_005\');"; + + + const char *SQL_ADD_FIELD_TABLE1 = "ALTER TABLE RDB_1 ADD COLUMN add_id INI"; + + const char *SQL_ADD_INDEX_TABLE2 = "CREATE INDEX name_index RDB_2 (name)"; + + const char *SQL_DROP_TABLE3 = "DROP TABLE RDB_3"; + + const char *SQL_DROP_CREATE_TABLE3 = "CREATE TABLE IF NOT EXISTS RDB_3(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT);"; + + const char *SQL_DROP_TABLE4 = "DROP TABLE RDB_4"; + + const char *SQL_DROP_CREATE_TABLE4 = "CREATE TABLE IF NOT EXISTS RDB_4(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, name CHAR(100));"; + + const char *SQL_JOURNAL_MODE = "PRAGMA journal_mode = DELETE;"; + + const char *SQL_SYNCHRONOUS_MODE = "PRAGMA synchronous = FULL;"; + + const char *SQL_INSERT_RDB5_TABLE = "INSERT INTO RDB_5 (id,name,age)" \ + "VALUES (1, \'rdb_005\', \'name_rdb5\');"; +} + +DBStatus DistributedRdbTestTools::GetOpenStoreStatus(const RelatetionalStoreManager *&manager, RelatetionalStoreDelegate *&delegate, + const RdbParameters ¶m) +{ + if (manager == nullptr) { + MST_LOG("%s GetRdbStore failed! manager nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + if (delegate != nullptr) { + delegate = nullptr; + } + DBStatus status = manager->OpenStore(param.path, param.storeId, delegate); + if (delegate == nullptr) { + MST_LOG("%s GetRdbStore failed! delegate nullptr.", TAG.c_str()); + } + retrun status; +} + +DBStatus DistributedRdbTestTools::GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, + const std::string &tableName) +{ + if (delegate == nullptr) { + MST_LOG("%s CreateDistributedTable failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + DBStatus status = delegate->CreateDistributedTable(tableName); + retrun status; +} + +DBStatus DistributedRdbTestTools::CloseStore(const RelatetionalStoreDelegate *&delegate) +{ + if (delegate == nullptr) { + MST_LOG("%s CloseStore failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + DBStatus status = delegate->CloseStore(); + return status; +} + +bool InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m) +{ + const std::string dbName = param.path + param.storeId + ".db"; + DBStributedDB::OS::RemoveFile(dbName); + int errCode = sqlite3_open(dbName, &db); + if (errCode1 != SQLITE_OK) { + MST_LOG("sqlite3_open Failed!"); + return false; + } + errCode = sqlite3_exec(db, "PRAGMA journal_mode = WAL;", 0, 0, 0); + if (errCode != SQLITE_OK) { + MST_LOG("PRAGMA journal_mode = WAL Failed!"); + return false; + } + return true; +} + +bool InitTableDataAndTRIGGER(const sqlite3 *&db) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + char *errMsg = nullptr; + int errCode1 = sqlite3_exec(db, SQL_CREATE_NORMAL_TABLE, nullptr, nullptr, &errMsg); + if (errCode1 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_CREATE_NORMAL_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + int errCode2 = sqlite3_exec(db, SQL_CREATE_CONSTRAINT_TABLE, nullptr, nullptr, &errMsg); + if (errCode2 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_CREATE_CONSTRAINT_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + int errCode3 = sqlite3_exec(db, SQL_CREATE_TRIGGER, nullptr, nullptr, &errMsg); + if (errCode3 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_CREATE_TRIGGER Failed(%s)", errMsg.c_str()); + return false; + } + + int errCode4 = sqlite3_exec(db, SQL_INSERT_NORMAL_TABLE, nullptr, nullptr, &errMsg); + if (errCode4 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_INSERT_NORMAL_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + for (int i = 1; i <= 33; i++) { + std::string str_0 = "RDB_" + i; + std::string str_1 = "CREATE TABLE IF NOT EXISTS " + std::string str_2 = "( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "name VARCHAR(100) NOT NULL, age VARCHAR(100) NOT NULL);"; + sprintf(str1,"%s%s",str0,str2); + + char *errMsg = nullptr; + int errCode1 = sqlite3_exec(db, str_1.c_str(), nullptr, nullptr, &errMsg); + if (errCode1 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec PresetTables33 Failed(%s)", errMsg.c_str()); + return false; + } + gtableNameList[i-1] = str_0; + } + return true; +} + +bool alterTableAttributes(const sqlite3 *&db) { + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + char *errMsg = nullptr; + int errCode = sqlite3_exec(db, SQL_ADD_FIELD_TABLE1, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_FIELD_TABLE1 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_ADD_INDEX_TABLE2, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_INDEX_TABLE2 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_TABLE3, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_TABLE3 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_CREATE_TABLE3, nullptr, nullptr, &errMsg); + if (errCode1 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_INDEX_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_TABLE4, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_TABLE4 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_CREATE_TABLE4, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_CREATE_TABLE4 Failed(%s)", errMsg.c_str()); + return false; + } + return true; +} + + +bool Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + int errCode = sqlite3_exec(db, sql_name, 0, 0, 0); + if (errCode != SQLITE_OK) { + MST_LOG("%s Failed!", sql_name); + return false; + } + return true; +} + +void CloseSqlite3Store(sqlite3 *&db) +{ + if (db != nullptr) { + sqlite3_close(db); + db = nullptr; + } +} \ No newline at end of file diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp new file mode 100644 index 000000000..3718f5378 --- /dev/null +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "relational_store_delegate.h" +#include "relational_store_manager.h" +#include "distributed_rdb_tools.h" + +using namespace std; +using namespace testing; +#if defined TESTCASES_USING_GTEST_EXT +using namespace testing::ext; +#endif +using namespace DistributedDB; +using namespace DistributedDBDataGenerator; + +// set sqlite3 rdb_A and create 33 dataTables + +namespace DistributedRelatinalDBExceptionOperation { +sqlite3 *gdb = nullptr; + +RelatetionalStoreManager *g_manager = nullptr; + +RelatetionalStoreDelegate *g_delegate = nullptr; + +RelatetionalStoreDelegate *g_delegate1 = nullptr; + +class DistributedRelatinalDBExceptionOperationTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +private: +}; + +void DistributedRelatinalDBExceptionOperationTest::SetUpTestCase(void) +{ + bool init1 = InitSqlite3Store(gdb, g_rdbParameter1); + ASSERT_EQ(init1, true); + bool init2 = InitTableDataAndTRIGGER(gdb); + ASSERT_EQ(init2, true); + g_manager = new (std::nothrow) RelatetionalStoreManager(g_rdbParameter1.appId, g_rdbParameter1.userId); + DBStatus status1 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate, g_rdbParameter1); + ASSERT_EQ(status1, DBStatus::OK); +} + +void DistributedRelatinalDBExceptionOperationTest::TearDownTestCase(void) +{ + CloseSqlite3Store(gdb); +} + +void DistributedRelatinalDBExceptionOperationTest::SetUp(void) +{ + UnitTest *test = UnitTest::GetInstance(); + ASSERT_NE(test, nullptr); + const TestInfo *testinfo = test->current_test_info(); + ASSERT_NE(testinfo, nullptr); + string testCaseName = string(testinfo->name()); + MST_LOG("[SetUp] test case %s is start to run", testCaseName.c_str()); +} + +void DistributedRelatinalDBExceptionOperationTest::TearDown(void) +{ + MST_LOG("TearDownTestCase after case."); +} + +/** + * @tc.name: dbPathException001 + * @tc.desc: db path does not exist, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException001, TestSize.Level4) +{ + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter6); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: dbPathException002 + * @tc.desc: db path unreadable, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException002, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = InitSqlite3Store(db, g_rdbParameter7); + ASSERT_EQ(init1, true); + bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IXUSR); + ASSERT_EQ(bpermission, true); + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter7); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); + bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + CloseSqlite3Store(db); +} + +/** + * @tc.name: dbPathException003 + * @tc.desc: db path unwritdable, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException003, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = InitSqlite3Store(db, g_rdbParameter8); + ASSERT_EQ(init1, true); + bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter8); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); + bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IWUSR | S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + CloseSqlite3Store(db); +} + +/** + * @tc.name: storeIdException001 + * @tc.desc: storeId is empty, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException001, TestSize.Level4) +{ + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter2); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: storeIdException002 + * @tc.desc: storeId value larger 128, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException002, TestSize.Level4) +{ + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter5); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: storeIdException003 + * @tc.desc: storeId is illegal, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException003, TestSize.Level4) +{ + DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter3); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: dbModeException001 + * @tc.desc: db mode isn't wal or SYNCHRONOUS full, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbModeException001, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = InitSqlite3Store(db, g_rdbParameter4); + ASSERT_EQ(init1, true); + bool res1 = Sqlite3ExecOpration(db, SQL_JOURNAL_MODE); + ASSERT_EQ(res1, true); + DBStatus status1 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + ASSERT_EQ(status1, DBStatus::NOT_SUPPORT); + CloseSqlite3Store(db); + + bool init2 = InitSqlite3Store(db, g_rdbParameter9); + ASSERT_EQ(init2, true); + bool res2 = Sqlite3ExecOpration(db, SQL_SYNCHRONOUS_MODE); + ASSERT_EQ(res2, true); + DBStatus status2 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); + CloseSqlite3Store(db); +} + +/** + * @tc.name: tableException001 + * @tc.desc: db hasn't non_table, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, NON_EXISTENT_TABLE); + ASSERT_EQ(status2, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: tableException002 + * @tc.desc: db create natrualbase_rdb_A relatinalTable, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException002, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, KEYWORD_START_TABLE); + ASSERT_EQ(status2, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: tableException003 + * @tc.desc: db NORMAL_RDB table has data, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException003, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, NORMAL_TABLE); + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); +} + +/** + * @tc.name: tableMulCreate001 + * @tc.desc: db create RDB_1 RelatinalTable, multiple createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableMulCreate001, TestSize.Level4) +{ + for (int i = 1; i <= 5; i++) { + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[0]); + ASSERT_EQ(status2, DBStatus::OK); + } +} + +/** + * @tc.name: mulTableCreate001 + * @tc.desc: dbA createDistributedTable 32 tables, successful + * then dbA createDistributedTable the 33'th table failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, mulTableCreate001, TestSize.Level4) +{ + for (auto tableName: gtableNameList) { + DBStatus status = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, tableName); + if (tableName == gtableNameList[32]) { + ASSERT_EQ(status, DBStatus::NOT_SUPPORT); + } else { + ASSERT_EQ(status, DBStatus::OK); + } + } +} + +/** + * @tc.name: mulTableCreate002 + * @tc.desc: dbA createDistributedTable RDB_1...20 tables, And + * dbB createDistributedTable RDB_21...32 tables,successful + * then dbA createDistributedTable the 33'th table failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, mulTableCreate002, TestSize.Level4) +{ + RelatetionalStoreDelegate *delegateA = nullptr; + RelatetionalStoreDelegate *delegateB = nullptr; + DBStatus status1 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, delegateA, g_rdbParameter1); + ASSERT_EQ(status1, DBStatus::OK); + DBStatus status2 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, delegateB, g_rdbParameter1); + ASSERT_EQ(status2, DBStatus::OK); + for (int index = 0; index < gtableNameList.size(); index++) { + if (index < 20) { + DBStatus status = DistributedRdbTestTools::GetCreateDistributedTableStatus(delegateA, gtableNameList[index]); + ASSERT_EQ(status2, DBStatus::OK); + } else if (index >= 20 && index < 32) + { + DBStatus status = DistributedRdbTestTools::GetCreateDistributedTableStatus(delegateB, gtableNameList[index]); + ASSERT_EQ(status2, DBStatus::OK); + } else { + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); + } + } +} + +/** + * @tc.name: relatinalTable001 + * @tc.desc: db has RDB_1 relatinalTable, then alter RDB_1 and createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, relatinalTable001, TestSize.Level4) +{ + bool result = alterTableAttributes(gdb); + ASSERT_EQ(result, true); + for (int i = 0; i < 4; i++) { + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[i]); + ASSERT_EQ(status2, DBStatus::OK); + } +} + +/** + * @tc.name: constraintTable001 + * @tc.desc: db has constraint table, then createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, constraintTable001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, CONSTRAINT_TABLE); + ASSERT_EQ(status2, DBStatus::OK); +} + +/** + * @tc.name: constraintTable001 + * @tc.desc: db has relatinalTable RDB_5, then add data in local RDB_5 and createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRelatinalDBExceptionOperationTest, constraintTable001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + ASSERT_EQ(status2, DBStatus::OK); + bool res1 = Sqlite3ExecOpration(db, SQL_INSERT_RDB5_TABLE); + ASSERT_EQ(res1, true); + DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + ASSERT_EQ(status2, DBStatus::OK); +} +} -- Gitee From c1c7e4db33b668609d98220048ae4146e988ce28 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Sat, 25 Dec 2021 10:41:58 +0800 Subject: [PATCH 18/53] Some UT. Add new file to BUILD.gn. Change the prefix of table. Signed-off-by: lidwchn --- .../libs/distributeddb/BUILD.gn | 1 + .../common/include/db_constant.h | 4 + .../distributeddb/common/include/db_errno.h | 2 +- .../distributeddb/common/src/db_constant.cpp | 2 + .../relational/relational_sync_able_storage.h | 7 +- .../src/relational_sync_able_storage.cpp | 99 ++--- .../src/sqlite/sqlite_query_helper.cpp | 126 ++++++- .../storage/src/sqlite/sqlite_query_helper.h | 5 +- .../sqlite/sqlite_single_ver_continue_token.h | 5 - ...e_single_ver_relational_continue_token.cpp | 122 +++++++ ...ite_single_ver_relational_continue_token.h | 56 +++ ...single_ver_relational_storage_executor.cpp | 251 ++++--------- ...e_single_ver_relational_storage_executor.h | 16 +- .../sqlite_single_ver_storage_executor.cpp | 6 +- ...lite_single_ver_storage_executor_cache.cpp | 2 +- .../storage/src/sqlite/sqlite_utils.cpp | 19 +- .../libs/distributeddb/test/BUILD.gn | 1 + ...distributeddb_relational_get_data_test.cpp | 344 ++++++++++++++++++ 18 files changed, 777 insertions(+), 291 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp create mode 100644 services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h create mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index 924097bb6..29f9c8c04 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -182,6 +182,7 @@ ohos_shared_library("distributeddb") { "storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp", "storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "storage/src/sqlite/sqlite_single_ver_result_set.cpp", "storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index 9c5d0e65b..fc0a4e6bb 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -112,6 +112,10 @@ public: static constexpr int DOUBLE_PRECISION = 15; static constexpr int MAX_DISTRIBUTED_TABLE_COUNT = 32; + + // For relational + static const std::string RELATIONAL_PREFIX; + static const std::string TIMESTAMP_ALIAS; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h index 5dd9a6e1f..f85331478 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h @@ -115,7 +115,7 @@ constexpr int E_SYSTEM_API_ADAPTER_CALL_FAILED = (E_BASE + 91); // Adapter call constexpr int E_NOT_NEED_DELETE_MSG = (E_BASE + 92); // not need delete msg, will be delete by sliding window receiver constexpr int E_SLIDING_WINDOW_SENDER_ERR = (E_BASE + 93); // sliding window sender err constexpr int E_SLIDING_WINDOW_RECEIVER_INVALID_MSG = (E_BASE + 94); // sliding window receiver invalid msg -constexpr int E_IGNOR_DATA = (E_BASE + 95); // ignore the data changed by other devices and ignore the same data. +constexpr int E_IGNORE_DATA = (E_BASE + 95); // ignore the data changed by other devices and ignore the same data. constexpr int E_FORBID_CACHEDB = (E_BASE + 96); // such after rekey can not check passwd due to file control. constexpr int E_INTERCEPT_DATA_FAIL = (E_BASE + 97); // Intercept push data failed. constexpr int E_INVALID_COMPRESS_ALGO = (E_BASE + 98); // The algo is defined, but there's no implement for the algo. diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp index 269b46436..eb90c8a4a 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp @@ -65,4 +65,6 @@ const std::string DBConstant::TRIGGER_REFERENCES_OLD = "OLD."; const std::string DBConstant::UPDATE_META_FUNC = "update_meta_within_trigger"; const std::string DBConstant::SYSTEM_TABLE_PREFIX = "naturalbase_rdb_"; +const std::string DBConstant::RELATIONAL_PREFIX = SYSTEM_TABLE_PREFIX + "aux_"; +const std::string DBConstant::TIMESTAMP_ALIAS = RELATIONAL_PREFIX + "timestamp"; } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index 37ef57ada..05bb9b1bf 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -19,7 +19,7 @@ #include "relational_db_sync_interface.h" #include "sqlite_single_relational_storage_engine.h" -#include "sqlite_single_ver_continue_token.h" +#include "sqlite_single_ver_relational_continue_token.h" namespace DistributedDB { class RelationalSyncAbleStorage : public RelationalDBSyncInterface, public virtual RefObject { @@ -67,9 +67,6 @@ public: int GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; - // Release the continue token of getting data. - void ReleaseContinueToken(ContinueToken &continueStmtToken) const override; - int PutSyncDataWithQuery(const QueryObject &object, const std::vector &entries, const DeviceID &deviceName) override; @@ -112,7 +109,7 @@ private: int SetMaxTimeStamp(TimeStamp timestamp); // get - int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerContinueToken *&continueStmtToken, + int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerRelationalContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const; // put diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index c8082bad6..3f18bdd04 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -196,15 +196,6 @@ const KvDBProperties &RelationalSyncAbleStorage::GetDbProperties() const return properties; } -static void ReleaseKvEntries(std::vector &entries) -{ - for (auto &itemEntry : entries) { - delete itemEntry; - itemEntry = nullptr; - } - entries.clear(); -} - static int GetKvEntriesByDataItems(std::vector &entries, std::vector &dataItems) { int errCode = E_OK; @@ -213,7 +204,7 @@ static int GetKvEntriesByDataItems(std::vector &entries, std if (entry == nullptr) { errCode = -E_OUT_OF_MEMORY; LOGE("GetKvEntries failed, errCode:%d", errCode); - ReleaseKvEntries(entries); + SingleVerKvEntry::Release(entries); break; } entry->SetEntryData(std::move(item)); @@ -237,7 +228,7 @@ static constexpr float QUERY_SYNC_THRESHOLD = 0.50; static bool CanHoldDeletedData(const std::vector &dataItems, const DataSizeSpecInfo &dataSizeInfo, size_t appendLen) { - bool reachThreshold = false; + bool reachThreshold = (dataItems.size() >= dataSizeInfo.packetSize); for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { blockSize += GetDataItemSerialSize(dataItems[i], appendLen); reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); @@ -246,7 +237,7 @@ static bool CanHoldDeletedData(const std::vector &dataItems, const Dat } static void ProcessContinueTokenForQuerySync(const std::vector &dataItems, int &errCode, - SQLiteSingleVerContinueToken *&token) + SQLiteSingleVerRelationalContinueToken *&token) { if (errCode != -E_UNFINISHED) { // Error happened or get data finished. Token should be cleared. delete token; @@ -261,18 +252,7 @@ static void ProcessContinueTokenForQuerySync(const std::vector &dataIt token = nullptr; return; } - - TimeStamp nextBeginTime = dataItems.back().timeStamp + 1; - if (nextBeginTime > INT64_MAX) { - nextBeginTime = INT64_MAX; - } - bool getDeleteData = ((dataItems.back().flag & DataItem::DELETE_FLAG) != 0); - if (getDeleteData) { - token->FinishGetQueryData(); - token->SetDeletedNextBeginTime("", nextBeginTime); - } else { - token->SetNextBeginTime("", nextBeginTime); - } + token->SetNextBeginTime(dataItems.back()); } /** @@ -280,7 +260,7 @@ static void ProcessContinueTokenForQuerySync(const std::vector &dataIt * If error happened, token will be deleted here. */ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &dataItems, - SQLiteSingleVerContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const + SQLiteSingleVerRelationalContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const { if (storageEngine_ == nullptr) { return -E_INVALID_DB; @@ -298,36 +278,18 @@ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &da goto ERROR; } - // Get query data. - if (!continueStmtToken->IsGetQueryDataFinished()) { - LOGD("[SingleVerNStore] Get query data between %llu and %llu.", continueStmtToken->GetQueryBeginTime(), - continueStmtToken->GetQueryEndTime()); - errCode = handle->GetSyncDataByQuery( - dataItems, Parcel::GetAppendedLen(), continueStmtToken->GetQuery(), dataSizeInfo, - // need modify - std::make_pair(continueStmtToken->GetQueryBeginTime(), continueStmtToken->GetQueryEndTime())); - } - - // Get query data finished. - if (errCode == E_OK || errCode == -E_FINISHED) { - // Clear query timeRange. - continueStmtToken->FinishGetQueryData(); - - // Get delete time next. - if (!continueStmtToken->IsGetDeletedDataFinished() && - CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())) { - LOGD("[SingleVerNStore] Get deleted data between %llu and %llu.", continueStmtToken->GetDeletedBeginTime(), - continueStmtToken->GetDeletedEndTime()); - errCode = handle->GetDeletedSyncDataByTimestamp( - dataItems, Parcel::GetAppendedLen(), - // need modify - continueStmtToken->GetDeletedBeginTime(), continueStmtToken->GetDeletedEndTime(), dataSizeInfo); + do { + errCode = handle->GetSyncDataByQuery(dataItems, + Parcel::GetAppendedLen(), + dataSizeInfo, + std::bind(&SQLiteSingleVerRelationalContinueToken::GetStatement, *continueStmtToken, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + if (errCode == -E_FINISHED) { + continueStmtToken->FinishGetData(); + errCode = continueStmtToken->IsGetAllDataFinished() ? E_OK : -E_UNFINISHED; } - } - - if (errCode == -E_FINISHED) { - errCode = E_OK; - } + } while (errCode == -E_UNFINISHED && + CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())); ERROR: if (errCode != -E_UNFINISHED && errCode != E_OK) { // Error happened. @@ -348,13 +310,24 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return -E_INVALID_ARGS; } - auto token = new (std::nothrow) SQLiteSingleVerContinueToken(timeRange, query); // release in sync module + auto token = new (std::nothrow) SQLiteSingleVerRelationalContinueToken(timeRange, query); if (token == nullptr) { LOGE("[SingleVerNStore] Allocate continue token failed."); return -E_OUT_OF_MEMORY; } - int innerCode; + continueStmtToken = static_cast(token); + return GetSyncDataNext(entries, continueStmtToken, dataSizeInfo); +} + +int RelationalSyncAbleStorage::GetSyncDataNext(std::vector &entries, + ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const +{ + auto token = static_cast(continueStmtToken); + if (!token->CheckValid()) { + return -E_INVALID_ARGS; + } + std::vector dataItems; int errCode = GetSyncDataForQuerySync(dataItems, token, dataSizeInfo); if (errCode != E_OK && errCode != -E_UNFINISHED) { // The code need be sent to outside except new error happened. @@ -362,7 +335,7 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return errCode; } - innerCode = GetKvEntriesByDataItems(entries, dataItems); + int innerCode = GetKvEntriesByDataItems(entries, dataItems); if (innerCode != E_OK) { errCode = innerCode; delete token; @@ -372,18 +345,6 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return errCode; } -int RelationalSyncAbleStorage::GetSyncDataNext(std::vector &entries, - ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const -{ - return E_OK; -} - -// Release the continue token of getting data. -void RelationalSyncAbleStorage::ReleaseContinueToken(ContinueToken &continueStmtToken) const -{ - return; -} - int RelationalSyncAbleStorage::PutSyncDataWithQuery(const QueryObject &object, const std::vector &entries, const DeviceID &deviceName) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp index 2053fb76b..212b13f30 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp @@ -83,6 +83,38 @@ std::string FieldValue2String(const FieldValue &val, QueryValueType type) return ""; } } + +std::string GetSelectAndFromClauseForRDB(const std::string &tableName) +{ + return "SELECT b.data_key," + "b.device," + "b.ori_device," + "b.timestamp as " + DBConstant::TIMESTAMP_ALIAS + "," + "b.wtimestamp," + "b.flag," + "b.hash_key," + "a.* " + "FROM " + tableName + " AS a INNER JOIN " + DBConstant::RELATIONAL_PREFIX + tableName + "_log AS b " + "ON a.rowid=b.data_key "; +} + +std::string GetTimeRangeClauseForRDB() +{ + return " AND (" + DBConstant::TIMESTAMP_ALIAS + ">=? AND " + DBConstant::TIMESTAMP_ALIAS + "=? AND " + DBConstant::TIMESTAMP_ALIAS + "=? AND naturalbase_rdb_timestamp=? AND naturalbase_rdb_timestamp>> mulDevTableTimeRange_; - std::map>> mulDevTableDeletedTimeRange_; }; } // namespace DistributedDB #endif // SQLITE_SINGLE_VER_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp new file mode 100644 index 000000000..512a9a46e --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "sqlite_single_ver_relational_continue_token.h" +#include "sqlite_utils.h" + +namespace DistributedDB { +SQLiteSingleVerRelationalContinueToken::SQLiteSingleVerRelationalContinueToken( + const SyncTimeRange &timeRange, const QueryObject &object) + : isGettingDeletedData_(false), queryObj_(object), tableName_(object.GetTableName()), timeRange_(timeRange) +{} + +bool SQLiteSingleVerRelationalContinueToken::CheckValid() const +{ + bool isValid = (magicBegin_ == MAGIC_BEGIN && magicEnd_ == MAGIC_END); + if (!isValid) { + LOGE("Invalid continue token."); + } + return isValid; +} + +int SQLiteSingleVerRelationalContinueToken::GetStatement(sqlite3 *db, sqlite3_stmt *&stmt, bool &isGettingDeletedData) +{ + isGettingDeletedData = isGettingDeletedData_; + if (isGettingDeletedData) { + return GetDeletedDataStmt(db, stmt); + } + return GetQuerySyncStatement(db, stmt); +} + +void SQLiteSingleVerRelationalContinueToken::SetNextBeginTime(const DataItem &theLastItem) +{ + TimeStamp nextBeginTime = theLastItem.timeStamp + 1; + if (nextBeginTime > INT64_MAX) { + nextBeginTime = INT64_MAX; + } + if (!isGettingDeletedData_) { + timeRange_.beginTime = nextBeginTime; + return; + } + if ((theLastItem.flag & DataItem::DELETE_FLAG) != 0) { // The last one could be non-deleted. + timeRange_.deleteBeginTime = nextBeginTime; + } +} + +void SQLiteSingleVerRelationalContinueToken::FinishGetData() +{ + if (isGettingDeletedData_) { + timeRange_.deleteEndTime = 0; + return; + } + isGettingDeletedData_ = true; + timeRange_.endTime = 0; + return; +} + +bool SQLiteSingleVerRelationalContinueToken::IsGetAllDataFinished() const +{ + return timeRange_.beginTime >= timeRange_.endTime && timeRange_.deleteBeginTime >= timeRange_.deleteEndTime; +} + +int SQLiteSingleVerRelationalContinueToken::GetQuerySyncStatement(sqlite3 *db, sqlite3_stmt *&stmt) +{ + int errCode = E_OK; + SqliteQueryHelper helper = queryObj_.GetQueryHelper(errCode); + if (errCode != E_OK) { + return errCode; + } + return helper.GetRelationalQueryStatement(db, timeRange_.beginTime, timeRange_.endTime, stmt); +} + +int SQLiteSingleVerRelationalContinueToken::GetDeletedDataStmt(sqlite3 *db, sqlite3_stmt *&stmt) const +{ + // get stmt + int errCode = E_OK; + const std::string sql = GetDeletedDataSQL(); + errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + goto ERROR; + } + + // bind stmt + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 1, timeRange_.deleteBeginTime); // 1 means begin time + if (errCode != E_OK) { + goto ERROR; + } + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 2, timeRange_.deleteEndTime); // 2 means end time + if (errCode != E_OK) { + goto ERROR; + } + return errCode; + +ERROR: + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} + +const QueryObject &SQLiteSingleVerRelationalContinueToken::GetQuery() const +{ + return queryObj_; +} + +std::string SQLiteSingleVerRelationalContinueToken::GetDeletedDataSQL() const +{ + std::string tableName = DBConstant::RELATIONAL_PREFIX + tableName_ + "_log"; + return "SELECT * FROM " + tableName + + " WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03 = 0x03) ORDER BY timestamp ASC;"; +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h new file mode 100644 index 000000000..a6542b488 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H +#define SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H +#ifdef RELATIONAL_STORE +#include +#include + +#include "db_types.h" +#include "query_object.h" +#include "single_ver_kvdb_sync_interface.h" + +namespace DistributedDB { +class SQLiteSingleVerRelationalContinueToken { +public: + SQLiteSingleVerRelationalContinueToken(const SyncTimeRange &timeRange, const QueryObject &queryObject); + ~SQLiteSingleVerRelationalContinueToken() = default; + + // Check the magic number at the beginning and end of the SQLiteSingleVerRelationalContinueToken. + bool CheckValid() const; + // The statement should be release by the caller. + int GetStatement(sqlite3 *db, sqlite3_stmt *&stmt, bool &isGettingDeletedData); + void SetNextBeginTime(const DataItem &theLastItem); + void FinishGetData(); + bool IsGetAllDataFinished() const; + const QueryObject &GetQuery() const; + +private: + std::string GetDeletedDataSQL() const; + int GetQuerySyncStatement(sqlite3 *db, sqlite3_stmt *&stmt); + int GetDeletedDataStmt(sqlite3 *db, sqlite3_stmt *&stmt) const; + + static const unsigned int MAGIC_BEGIN = 0x600D0AC7; // for token guard + static const unsigned int MAGIC_END = 0x0AC7600D; // for token guard + unsigned int magicBegin_ = MAGIC_BEGIN; + int isGettingDeletedData_ = false; + QueryObject queryObj_; + const std::string &tableName_; + SyncTimeRange timeRange_; + unsigned int magicEnd_ = MAGIC_END; +}; +} // namespace DistributedDB +#endif // RELATIONAL_STORE +#endif // SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 9fa4e7183..56bb8cc70 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -96,7 +96,7 @@ int SQLiteSingleVerRelationalStorageExecutor::Rollback() return errCode; } -int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) +int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(const QueryObject &query) { int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); @@ -106,50 +106,6 @@ int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) return errCode; } -// binding index just for the get sync data sql -static const int BIND_BEGIN_STAMP_INDEX = 1; -static const int BIND_END_STAMP_INDEX = 2; - -int SQLiteSingleVerRelationalStorageExecutor::PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, - sqlite3_stmt *&statement, bool getDeletedData) const -{ - if (dbHandle_ == nullptr) { - return -E_INVALID_DB; - } - - const std::string SELECT_SYNC_DELETED_ENTRIES_SQL = - "SELECT * FROM naturalbase_rdb_aux_" + table_.GetTableName() + - "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x03) ORDER BY timestamp ASC;"; - const std::string SELECT_SYNC_ENTRIES_SQL = - "SELECT * FROM naturalbase_rdb_aux_" + table_.GetTableName() + - "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x02=0x02) ORDER BY timestamp ASC;"; - - const std::string sql = (getDeletedData ? SELECT_SYNC_DELETED_ENTRIES_SQL : SELECT_SYNC_ENTRIES_SQL); - int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); - if (errCode != E_OK) { - LOGE("Prepare the sync entries statement error:%d", errCode); - return errCode; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_BEGIN_STAMP_INDEX, begin); - if (errCode != E_OK) { - goto ERROR; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_END_STAMP_INDEX, end); - if (errCode != E_OK) { - goto ERROR; - } - -ERROR: - if (errCode != E_OK) { - LOGE("Bind the timestamp for getting sync data error:%d", errCode); - SQLiteUtils::ResetStatement(statement, true, errCode); - } - - return CheckCorruptedStatus(errCode); -} - static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) { switch (type) { @@ -260,7 +216,6 @@ static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) return errCode; } logInfo.originDev = std::string(oriDev.begin(), oriDev.end()); - logInfo.timestamp = static_cast(sqlite3_column_int64(logStatement, 3)); logInfo.wTimeStamp = static_cast(sqlite3_column_int64(logStatement, 4)); logInfo.flag = static_cast(sqlite3_column_int64(logStatement, 5)); @@ -276,68 +231,6 @@ static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) return errCode; } -static sqlite3_stmt *GetDataStmtByPK(sqlite3 *db, const std::string &tableName, int rowId) -{ - std::string sql = "select * from " + tableName + " where rowId=?;"; - sqlite3_stmt *statement = nullptr; - int errCode = SQLiteUtils::GetStatement(db, sql, statement); - if (errCode != E_OK) { - LOGE("[data by rowid statement] Get statement fail!"); - return nullptr; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, 1, rowId); - if (errCode != E_OK) { - SQLiteUtils::ResetStatement(statement, true, errCode); - return nullptr; - } - - return statement; -} - - -int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *logStatement, DataItem &dataItem) const -{ - std::map colInfos = table_.GetFields(); - RowDataWithLog data; - - int errCode = GetLogData(logStatement, data.logInfo); - if (errCode != E_OK) { - LOGE("relational data value transfer to kv fail"); - return errCode; - } - - // dataKey is rowid by pk - // only get local device data - sqlite3_stmt *stmt = GetDataStmtByPK(dbHandle_, table_.GetTableName(), data.logInfo.dataKey); - - errCode = SQLiteUtils::StepWithRetry(stmt, false); - if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW) && errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGE("[SQLiteSingleVerRelationalStorageExecutor] Step fail errCode:%d", errCode); - return errCode; - } - - std::vector fieldInfos; - for (const auto &col: colInfos) { - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - break; - } - DataValue value; - GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId()); - - fieldInfos.push_back(col.second); - data.rowData.push_back(value); - } - - SQLiteUtils::ResetStatement(stmt, true, errCode); - - errCode = DataTransformer::SerializeDataItem(data, fieldInfos, dataItem); - if (errCode != E_OK) { - LOGE("relational data value transfer to kv fail"); - } - return errCode; -} - static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) { // timestamp and local flag: 3 * uint64_t, version(uint32_t), key, value, origin dev and the padding size. @@ -349,54 +242,8 @@ static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) return dataSize; } -int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataItems(std::vector &dataItems, - sqlite3_stmt *logStatement, size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const -{ - int errCode; - size_t dataTotalSize = 0; - do { - DataItem item; - errCode = SQLiteUtils::StepWithRetry(logStatement, isMemDb_); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - errCode = GetDataItemForSync(logStatement, item); - } else { - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); - errCode = -E_FINISHED; - } else { - LOGE("Get sync data error:%d", errCode); - } - break; - } - - // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. - dataTotalSize += GetDataItemSerialSize(item, appendLength); - if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || - dataItems.size() >= dataSizeInfo.packetSize) { - errCode = -E_UNFINISHED; - break; - } else { - dataItems.push_back(std::move(item)); - } - } while (true); - return errCode; -} - -int SQLiteSingleVerRelationalStorageExecutor::GetDeletedSyncDataByTimestamp(std::vector &dataItems, - size_t appendLength, TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const -{ - sqlite3_stmt *logStatement = nullptr; - int errCode = PrepareForSyncDataByTime(begin, end, logStatement, true); - if (errCode != E_OK) { - return errCode; - } - - errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); - SQLiteUtils::ResetStatement(logStatement, true, errCode); - return CheckCorruptedStatus(errCode); -} - -static const std::string SELECT_META_VALUE_SQL = "SELECT value FROM naturalbase_rdb_aux_metadata WHERE key=?;"; +static const std::string SELECT_META_VALUE_SQL = + "SELECT value FROM " + DBConstant::RELATIONAL_PREFIX + "metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &value) const { sqlite3_stmt *statement = nullptr; @@ -424,7 +271,8 @@ int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &v return errCode; } -static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO naturalbase_rdb_aux_metadata VALUES(?,?);"; +static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + + "metadata VALUES(?,?);"; int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const { sqlite3_stmt *statement = nullptr; @@ -453,7 +301,8 @@ ERROR: return errCode; } -static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM naturalbase_rdb_aux_metadata WHERE key=?;"; +static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const { sqlite3_stmt *statement = nullptr; @@ -480,7 +329,7 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector=? AND key<=?;"; + "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + "metadata WHERE key>=? AND key<=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const { sqlite3_stmt *statement = nullptr; @@ -527,7 +376,8 @@ static int GetAllKeys(sqlite3_stmt *statement, std::vector &keys) return errCode; } -static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM naturalbase_rdb_aux_metadata;"; + +static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM " + DBConstant::RELATIONAL_PREFIX + "metadata;"; int SQLiteSingleVerRelationalStorageExecutor::GetAllMetaKeys(std::vector &keys) const { sqlite3_stmt *statement = nullptr; @@ -545,7 +395,7 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj const std::string &deviceName, sqlite3_stmt *&logStmt) const { std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "naturalbase_rdb_aux_" + object.GetTableName() + "_log"; + const std::string tableName = DBConstant::RELATIONAL_PREFIX + object.GetTableName() + "_log"; std::string dataFormat = "?, '" + deviceName + "', ?, ?, ?, ?, ?"; std::string sql = "INSERT OR REPLACE INTO " + tableName + @@ -564,7 +414,7 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryOb // naturalbase_rdb_aux_userTableName_deviceHash // tableName std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "naturalbase_rdb_aux_" + object.GetTableName() + "_" + + const std::string tableName = DBConstant::RELATIONAL_PREFIX + object.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); @@ -602,7 +452,8 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen Key hashKey; (void)DBCommon::CalcValueHash(dataItem.key, hashKey); std::string hash = std::string(hashKey.begin(), hashKey.end()); - std::string sql = "select * from naturalbase_rdb_aux_" + table_.GetTableName() + "_log where hash_key = ?;"; + std::string sql = "select * from " + DBConstant::RELATIONAL_PREFIX + table_.GetTableName() + + "_log where hash_key = ?;"; sqlite3_stmt *queryStmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, queryStmt); if (errCode != E_OK) { @@ -660,7 +511,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem) { std::string devName = DBCommon::TransferHashString(dataItem.dev); - const std::string tableName = "naturalbase_rdb_aux_" + table_.GetTableName() + "_" + + const std::string tableName = DBConstant::RELATIONAL_PREFIX + table_.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); std::string hashKey = std::string(dataItem.hashKey.begin(), dataItem.hashKey.end()); std::string sql = "DELETE FROM " + tableName + " WHERE calc_hash(" + table_.GetPrimaryKey() + ")=" + hashKey + ";"; @@ -780,40 +631,72 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncItems(const QueryObject &o return errCode; } -static int GetLogInfoStatement(sqlite3 *dbHandle, const std::string &tableName, - uint64_t beginTime, uint64_t endTime, sqlite3_stmt *&statement) +int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *stmt, DataItem &dataItem, + bool isGettingDeletedData) const { - std::string sql = "select * from naturalbase_rdb_aux_" + tableName + - "_log where flag=0x02 AND timestamp>=? AND timestamp fieldInfos; + if (!isGettingDeletedData) { + for (const auto &col: table_.GetFields()) { + LOGD("[GetDataItemForSync] field:%s type:%d cid:%d", col.second.GetFieldName().c_str(), + col.second.GetStorageType(), col.second.GetColumnId() + 7); + DataValue value; + GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId() + 7); + fieldInfos.push_back(col.second); + data.rowData.push_back(value); + } } - errCode = SQLiteUtils::BindInt64ToStatement(statement, 2, endTime); + errCode = DataTransformer::SerializeDataItem(data, fieldInfos, dataItem); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + LOGE("relational data value transfer to kv fail"); } return errCode; } int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, - QueryObject query, const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const + const DataSizeSpecInfo &dataSizeInfo, std::function getStmt) { - sqlite3_stmt *logStatement = nullptr; - int errCode = GetLogInfoStatement(dbHandle_, query.GetTableName(), timeRange.first, timeRange.second, logStatement); - if (errCode == E_OK) { - errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); + sqlite3_stmt *stmt = nullptr; + bool isGettingDeletedData = false; + int errCode = getStmt(dbHandle_, stmt, isGettingDeletedData); + if (errCode != E_OK) { + return errCode; } - SQLiteUtils::ResetStatement(logStatement, true, errCode); + + size_t dataTotalSize = 0; + do { + DataItem item; + errCode = SQLiteUtils::StepWithRetry(stmt, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + errCode = GetDataItemForSync(stmt, item, isGettingDeletedData); + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); + errCode = -E_FINISHED; + break; + } else { + LOGE("Get sync data error:%d", errCode); + break; + } + + // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. + dataTotalSize += GetDataItemSerialSize(item, appendLength); + if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || + dataItems.size() >= dataSizeInfo.packetSize) { + errCode = -E_UNFINISHED; + break; + } else { + dataItems.push_back(std::move(item)); + } + } while (true); + + SQLiteUtils::ResetStatement(stmt, true, errCode); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h index 66769f825..ba0057ae1 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -16,8 +16,9 @@ #define SQLITE_SINGLE_VER_RELATIONAL_STORAGE_EXECUTOR_H #ifdef RELATIONAL_STORE -#include "macro_utils.h" +#include "data_transformer.h" #include "db_types.h" +#include "macro_utils.h" #include "sqlite_utils.h" #include "sqlite_storage_executor.h" #include "relational_store_delegate.h" @@ -38,13 +39,11 @@ public: int Commit(); int Rollback(); - int SetTableInfo(QueryObject query); + int SetTableInfo(const QueryObject &query); // For Get sync data - int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, QueryObject query, - const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const; - int GetDeletedSyncDataByTimestamp(std::vector &dataItems, size_t appendLength, - TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const; + int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, + const DataSizeSpecInfo &dataSizeInfo, std::function getStmt); // operation of meta data int GetKvData(const Key &key, Value &value) const; @@ -65,10 +64,7 @@ private: int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData) const; - int GetSyncDataItems(std::vector &dataItems, sqlite3_stmt *statement, - size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const; - - int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem) const; + int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem, bool isGettingDeletedData) const; int SaveSyncDataItems(const QueryObject &object, std::vector &dataItems, const std::string &deviceName, TimeStamp &timeStamp); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index 9d3e9a2ff..84d5b2348 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -1298,9 +1298,9 @@ int SQLiteSingleVerStorageExecutor::PrepareForNotifyConflictAndObserver(DataItem LOGD("[SingleVerExe] Ignore the sync data."); if (isSyncMigrating_) { ResetForMigrateCacheData(); - return -E_IGNOR_DATA; + return -E_IGNORE_DATA; } - return ResetSaveSyncStatements(-E_IGNOR_DATA); + return ResetSaveSyncStatements(-E_IGNORE_DATA); } notify.dataStatus = JudgeSyncSaveType(dataItem, notify.getData, deviceInfo.deviceName, isHashKeyExisted); @@ -1343,7 +1343,7 @@ int SQLiteSingleVerStorageExecutor::SaveSyncDataItem(DataItem &dataItem, const D int errCode = PrepareForNotifyConflictAndObserver(dataItem, deviceInfo, notify); if (errCode != E_OK) { - if (errCode == -E_IGNOR_DATA) { + if (errCode == -E_IGNORE_DATA) { errCode = E_OK; } return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp index 872e87361..9559b8fb4 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp @@ -760,7 +760,7 @@ int SQLiteSingleVerStorageExecutor::PutIntoConflictAndCommitForMigrateCache(Data int errCode = PrepareForNotifyConflictAndObserver(dataItem, deviceInfo, notify); if (errCode != E_OK) { errCode = (errCode == -E_NOT_FOUND ? E_OK : errCode); - if (errCode == -E_IGNOR_DATA) { + if (errCode == -E_IGNORE_DATA) { notify.dataStatus.isDefeated = true; errCode = E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 906d2952c..e69117af7 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1305,7 +1305,7 @@ int SQLiteUtils::RegisterGetSysTime(sqlite3 *db) int SQLiteUtils::CreateRelationalMetaTable(sqlite3 *db) { std::string sql = - "CREATE TABLE IF NOT EXISTS naturalbase_rdb_aux_metadata(" \ + "CREATE TABLE IF NOT EXISTS " + DBConstant::RELATIONAL_PREFIX + "metadata(" \ "key BLOB PRIMARY KEY NOT NULL," \ "value BLOB);"; @@ -1320,21 +1320,21 @@ int SQLiteUtils::CreateRelationalMetaTable(sqlite3 *db) int SQLiteUtils::CreateRelationalLogTable(sqlite3 *db, const std::string &oriTableName) { std::string sql = - "CREATE TABLE IF NOT EXISTS naturalbase_rdb_aux_" + oriTableName + "_log(" \ + "CREATE TABLE IF NOT EXISTS " + DBConstant::RELATIONAL_PREFIX + oriTableName + "_log(" \ "data_key INT NOT NULL," \ "device BLOB," \ "ori_device BLOB," \ "timestamp INT NOT NULL," \ "wtimestamp INT NOT NULL," \ "flag INT NOT NULL," \ - "hash_key BLOB PRIMARY KEY NOT NULL);"; + "hash_key BLOB NOT NULL," + "PRIMARY KEY(device,hash_key));"; int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); if (errCode != E_OK) { LOGE("[SQLite] execute create table sql failed"); - return errCode; } - return E_OK; + return errCode; } template @@ -1352,7 +1352,7 @@ int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &tabl "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_INSERT AFTER INSERT \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t INSERT OR REPLACE INTO naturalbase_rdb_aux_%s_log \ + "\t INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + "%s_log \ (data_key, device, ori_device, timestamp, wtimestamp, flag, hash_key)" \ "VALUES (new.rowid, '%s', '%s', get_sys_time(), get_sys_time(), 0x02, calc_hash(new.%s));\n" \ "END;"; @@ -1363,7 +1363,7 @@ int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &tabl "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_UPDATE AFTER UPDATE \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t UPDATE naturalbase_rdb_aux_%s_log SET timestamp=get_sys_time(), device='%s' \ + "\t UPDATE " + DBConstant::RELATIONAL_PREFIX + "%s_log SET timestamp=get_sys_time(), device='%s' \ where hash_key=calc_hash(old.%s) and flag&0x10=0;\n" \ "END;"; updateTrigger = string_format(updateTrigger, table.GetTableName().c_str(), table.GetTableName().c_str(), @@ -1372,7 +1372,7 @@ int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &tabl "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_DELETE BEFORE DELETE \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t UPDATE naturalbase_rdb_aux_%s_log set flag=0x03,timestamp=get_sys_time() \ + "\t UPDATE " + DBConstant::RELATIONAL_PREFIX + "%s_log set flag=0x03,timestamp=get_sys_time() \ WHERE hash_key=calc_hash(old.%s);\n" \ "END;"; deleteTrigger = string_format(deleteTrigger, table.GetTableName().c_str(), @@ -1403,9 +1403,8 @@ int SQLiteUtils::CreateSameStuTable(sqlite3 *db, const std::string &oriTableName int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); if (errCode != E_OK) { LOGE("[SQLite] execute create table sql failed"); - return errCode; } - return E_OK; + return errCode; } int SQLiteUtils::RegisterFunction(sqlite3 *db, const std::string &funcName, int nArg, void *uData, TransactFunc &func) diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 22581c572..3ca3c7d54 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -185,6 +185,7 @@ ohos_source_set("src_file") { "../storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "../storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp", "../storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "../storage/src/sqlite/sqlite_single_ver_result_set.cpp", "../storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp new file mode 100644 index 000000000..db75f5c58 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include + +#include "data_transformer.h" +#include "db_common.h" +#include "db_constant.h" +#include "db_errno.h" +#include "db_types.h" +#include "log_print.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "generic_single_ver_kv_entry.h" +#include "kvdb_properties.h" +#include "relational_store_delegate.h" +#include "relational_store_instance.h" +#include "relational_store_manager.h" +#include "relational_store_sqlite_ext.h" +#include "relational_sync_able_storage.h" +#include "sqlite_relational_store.h" +#include "sqlite_utils.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { +string g_testDir; +string g_storePath; +const string g_tableName { "data" }; +DistributedDB::RelationalStoreManager g_mgr(APP_ID, USER_ID); +RelationalStoreDelegate *g_delegate = nullptr; + +void CreateDBAndTable() +{ + sqlite3 *db = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + sqlite3_close(db); + return; + } + + const string sql { "CREATE TABLE " + g_tableName + "(key BLOB PRIMARY KEY NOT NULL, value BLOB);" }; + char *zErrMsg = nullptr; + errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &zErrMsg); + if (errCode != SQLITE_OK) { + LOGE("sql error:%s",zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_close(db); +} + +int InitRelationalStore() +{ + RelationalStoreDelegate *delegate = nullptr; + DBStatus dbStatus = DBStatus::DB_ERROR; + g_mgr.OpenStore(g_storePath, RelationalStoreDelegate::Option {}, + [&delegate, &dbStatus] (DBStatus status, RelationalStoreDelegate *ptr) { + delegate = ptr; + dbStatus = status; + } + ); + if (dbStatus == DBStatus::OK) { + g_delegate = delegate; + g_delegate->CreateDistributedTable(g_tableName, RelationalStoreDelegate::TableOption()); + } + return dbStatus; +} + +int AddOrUpdateRecord(const Key &key, const Value &value) { + static const string SQL = "INSERT OR REPLACE INTO " + g_tableName + " VALUES(?,?);"; + sqlite3 *db = nullptr; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + goto END; + } + errCode = SQLiteUtils::GetStatement(db, SQL, statement); + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // 1 means key's index + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::BindBlobToStatement(statement, 2, value, true); // 2 means value's index + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::StepWithRetry(statement, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + +END: + SQLiteUtils::ResetStatement(statement, true, errCode); + sqlite3_close(db); + return errCode; +} + +int GetLogData(const Key &key, uint64_t &flag, TimeStamp ×tamp, const DeviceID &device = "") +{ + string tableName = g_tableName; + if (!device.empty()) { + } + const string sql = "SELECT timestamp, flag \ + FROM " + g_tableName + " as a, " + DBConstant::RELATIONAL_PREFIX + g_tableName + "_log as b \ + WHERE a.key=? AND a.rowid=b.data_key;"; + + sqlite3 *db = nullptr; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + goto END; + } + errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // 1 means key's index + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::StepWithRetry(statement, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + timestamp = static_cast(sqlite3_column_int64(statement, 0)); + flag = static_cast(sqlite3_column_int64(statement, 1)); + errCode = E_OK; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = -E_NOT_FOUND; + } + +END: + sqlite3_close(db); + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, + const std::string appId, const std::string &userId, DBProperties &properties) +{ + properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); + properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); + properties.SetStringProp(KvDBProperties::APP_ID, appId); + properties.SetStringProp(KvDBProperties::USER_ID, userId); + properties.SetStringProp(KvDBProperties::STORE_ID, storePath); + std::string identifier = userId + "-" + appId + "-" + storePath; + std::string hashIdentifier = DBCommon::TransferHashString(identifier); + properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); +} + +const RelationalSyncAbleStorage *GetRelationalStore() +{ + DBProperties properties; + InitStoreProp(RelationalStoreDelegate::Option {}, g_storePath, APP_ID, USER_ID, properties); + int errCode = E_OK; + auto store = RelationalStoreInstance::GetDataBase(properties, errCode); + if (store == nullptr) { + LOGE("Get db failed:%d", errCode); + return nullptr; + } + return static_cast(store)->GetStorageEngine(); +} +} + +class DistributedDBRelationalGetDataTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBRelationalGetDataTest::SetUpTestCase(void) +{ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + g_storePath = g_testDir + "/getDataTest.db"; + LOGI("The test db is:%s", g_testDir.c_str()); +} + +void DistributedDBRelationalGetDataTest::TearDownTestCase(void) +{} + +void DistributedDBRelationalGetDataTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + CreateDBAndTable(); +} + +void DistributedDBRelationalGetDataTest::TearDown(void) +{ + if (g_delegate != nullptr) { + EXPECT_EQ(g_mgr.CloseStore(g_delegate), DBStatus::OK); + EXPECT_EQ(g_mgr.DeleteStore(g_storePath), DBStatus::OK); + g_delegate = nullptr; + } + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error."); + } + return; +} + +/** + * @tc.name: LogTbl1 + * @tc.desc: When put sync data to relational store, trigger generate log. + * @tc.type: FUNC + * @tc.require: AR000GK58G + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, LogTbl1, TestSize.Level1) +{ + ASSERT_EQ(InitRelationalStore(), DBStatus::OK); + + /** + * @tc.steps: step1. Put data. + * @tc.expected: Succeed, return OK. + */ + Key insertKey = KEY_1; + Value insertValue = VALUE_1; + EXPECT_EQ(AddOrUpdateRecord(insertKey, insertValue), E_OK); + + /** + * @tc.steps: step2. Check log record. + * @tc.expected: Record exists. + */ + Value value; + uint64_t flag = 0; + TimeStamp timestamp1 = 0; + EXPECT_EQ(GetLogData(insertKey, flag, timestamp1), E_OK); + EXPECT_EQ(flag, DataItem::LOCAL_FLAG); + EXPECT_NE(timestamp1, 0ULL); +} + +/** + * @tc.name: GetSyncData1 + * @tc.desc: GetSyncData interface + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) +{ + ASSERT_EQ(InitRelationalStore(), DBStatus::OK); + + /** + * @tc.steps: step1. Put 500 records. + * @tc.expected: Succeed, return OK. + */ + const int RECORD_COUNT = 500; + for (int i = 0; i < RECORD_COUNT; ++i) { + string key = "key_" + to_string(i); + string value = "value_" + to_string(i); + EXPECT_EQ(AddOrUpdateRecord(Key(key.data(), key.data() + key.size()), + Value(value.data(), value.data() + value.size())), E_OK); + } + + /** + * @tc.steps: step2. Get all data. + * @tc.expected: Succeed and the count is right. + */ + auto store = GetRelationalStore(); + ContinueToken token = nullptr; + QueryObject query(Query::Select(g_tableName)); + std::vector entries; + DataSizeSpecInfo sizeInfo {MTU_SIZE, 50}; + + int errCode = store->GetSyncData(query, SyncTimeRange {}, sizeInfo, token, entries); + int count = entries.size(); + SingleVerKvEntry::Release(entries); + EXPECT_EQ(errCode, -E_UNFINISHED); + while (token != nullptr) { + errCode = store->GetSyncDataNext(entries, token, sizeInfo); + count += entries.size(); + SingleVerKvEntry::Release(entries); + EXPECT_TRUE(errCode == E_OK || errCode == -E_UNFINISHED); + } + EXPECT_EQ(count, RECORD_COUNT); +} + +/** + * @tc.name: GetQuerySyncData1 + * @tc.desc: GetSyncData interface. + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) +{ + ASSERT_EQ(InitRelationalStore(), DBStatus::OK); + + /** + * @tc.steps: step1. Put 100 records. + * @tc.expected: Succeed, return OK. + */ + const int RECORD_COUNT = 100; // 100 records. + for (int i = 0; i < RECORD_COUNT; ++i) { + string key = "k" + to_string(i); + string value = "v" + to_string(i); + EXPECT_EQ(AddOrUpdateRecord(Key(key.data(), key.data() + key.size()), + Value(value.data(), value.data() + value.size())), E_OK); + } + + /** + * @tc.steps: step2. Get data limit 80, offset 30. + * @tc.expected: Get 70 records. + */ + auto store = GetRelationalStore(); + ContinueToken token = nullptr; + const unsigned int LIMIT = 80; // limit as 80. + const unsigned int OFFSET = 30; // offset as 30. + const unsigned int EXPECT_COUNT = RECORD_COUNT - OFFSET; // expect 70 records. + QueryObject query(Query::Select(g_tableName).Limit(LIMIT, OFFSET)); + std::vector entries; + + int errCode = store->GetSyncData(query, SyncTimeRange {}, DataSizeSpecInfo {}, token, entries); + EXPECT_EQ(entries.size(), EXPECT_COUNT); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(token, nullptr); + SingleVerKvEntry::Release(entries); +} +#endif \ No newline at end of file -- Gitee From a6f50ba75b21dd727f7f15fbb7e758f8898998a5 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 30 Dec 2021 10:21:05 +0800 Subject: [PATCH 19/53] Fix compile issues in lower g++ version Signed-off-by: lianhuix --- .../include/relational_store_connection.h | 11 ++++----- .../src/relational_store_connection.cpp | 7 ++++++ ...single_ver_relational_storage_executor.cpp | 23 +++++++++---------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index 35f2d91e6..ac6cb4c13 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -34,11 +34,10 @@ public: const Query &query; bool wait = true; }; - RelationalStoreConnection() = default; - explicit RelationalStoreConnection(IRelationalStore *store) - { - store_ = store; - }; + + RelationalStoreConnection(); + + explicit RelationalStoreConnection(IRelationalStore *store); virtual ~RelationalStoreConnection() = default; @@ -61,7 +60,7 @@ protected: virtual int Pragma(int cmd, void *parameter); IRelationalStore *store_ = nullptr; - std::atomic isExclusive_ = false; + std::atomic isExclusive_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp index 4fc241f37..c04d81193 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp @@ -18,6 +18,13 @@ #include "sqlite_single_ver_relational_storage_executor.h" namespace DistributedDB { +RelationalStoreConnection::RelationalStoreConnection() : isExclusive_(false) +{} + +RelationalStoreConnection::RelationalStoreConnection(IRelationalStore *store) + : store_(store), isExclusive_(false) +{} + int RelationalStoreConnection::Pragma(int cmd, void *parameter) { return E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 56bb8cc70..bd20e5c69 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -242,10 +242,10 @@ static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) return dataSize; } -static const std::string SELECT_META_VALUE_SQL = - "SELECT value FROM " + DBConstant::RELATIONAL_PREFIX + "metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &value) const { + static const std::string SELECT_META_VALUE_SQL = "SELECT value FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_META_VALUE_SQL, statement); if (errCode != E_OK) { @@ -271,10 +271,10 @@ int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &v return errCode; } -static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + - "metadata VALUES(?,?);"; int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const { + static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + + "metadata VALUES(?,?);"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, INSERT_META_SQL, statement); if (errCode != E_OK) { @@ -301,10 +301,10 @@ ERROR: return errCode; } -static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + - "metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const { + static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_SQL, statement); if (errCode != E_OK) { @@ -328,10 +328,10 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector=? AND key<=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const { + static const std::string REMOVE_META_VALUE_BY_KEY_PREFIX_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key>=? AND key<=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_BY_KEY_PREFIX_SQL, statement); if (errCode != E_OK) { @@ -376,10 +376,9 @@ static int GetAllKeys(sqlite3_stmt *statement, std::vector &keys) return errCode; } - -static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM " + DBConstant::RELATIONAL_PREFIX + "metadata;"; int SQLiteSingleVerRelationalStorageExecutor::GetAllMetaKeys(std::vector &keys) const { + static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM " + DBConstant::RELATIONAL_PREFIX + "metadata;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_ALL_META_KEYS, statement); if (errCode != E_OK) { @@ -446,8 +445,8 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryOb return errCode; } -int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statement, - const DataItem &dataItem, TimeStamp &maxTimestamp) +int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statement, const DataItem &dataItem, + TimeStamp &maxTimestamp) { Key hashKey; (void)DBCommon::CalcValueHash(dataItem.key, hashKey); -- Gitee From 5f3ed182b566aaca267a12d494f7acda4e9e32ad Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 30 Dec 2021 15:33:51 +0800 Subject: [PATCH 20/53] Fix reviews Signed-off-by: lianhuix --- .../common/include/json_object.h | 2 +- .../distributeddb/common/src/db_common.cpp | 5 +---- .../distributeddb/common/src/json_object.cpp | 4 ++-- .../relational/relational_schema_object.cpp | 20 ++++++++----------- .../relational/relational_store_manager.h | 5 ----- .../relational/relational_store_manager.cpp | 12 +++-------- .../storage/include/db_properties.h | 3 +++ .../storage/src/db_properties.cpp | 19 ++++++++++++------ .../storage/src/irelational_store.h | 2 +- .../relational/sqlite_relational_store.cpp | 10 +++++----- ...single_ver_relational_storage_executor.cpp | 8 -------- .../sqlite_single_ver_storage_engine.cpp | 2 +- .../storage/src/sqlite/sqlite_utils.h | 1 - ...tributeddb_storage_index_optimize_test.cpp | 2 +- 14 files changed, 39 insertions(+), 56 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h index 0fcaf6478..ddad4bb79 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h @@ -65,7 +65,7 @@ public: int GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; int GetStringArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; - int GetObjectByFieldPath(const FieldPath &inPath, JsonObject &outArray) const; + int GetObjectByFieldPath(const FieldPath &inPath, JsonObject &outObj) const; // An empty fieldPath indicate the root, the outSubPath should be empty before call, we will not empty it at first. // If inPath is of multiple path, then outSubPath is combination of result of each inPath. diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp index de9192da3..c04b39406 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp @@ -290,9 +290,7 @@ std::string DBCommon::GenerateIdentifierId(const std::string &storeId, void DBCommon::SetDatabaseIds(KvDBProperties &properties, const std::string &appId, const std::string &userId, const std::string &storeId) { - properties.SetStringProp(KvDBProperties::APP_ID, appId); - properties.SetStringProp(KvDBProperties::USER_ID, userId); - properties.SetStringProp(KvDBProperties::STORE_ID, storeId); + properties.SetIdentifier(userId, appId, storeId); std::string oriStoreDir; std::string identifier = GenerateIdentifierId(storeId, appId, userId); if (properties.GetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, false)) { @@ -301,7 +299,6 @@ void DBCommon::SetDatabaseIds(KvDBProperties &properties, const std::string &app oriStoreDir = identifier; } std::string hashIdentifier = TransferHashString(identifier); - properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); std::string hashDir = TransferHashString(oriStoreDir); std::string hexHashDir = TransferStringToHex(hashDir); properties.SetStringProp(KvDBProperties::IDENTIFIER_DIR, hexHashDir); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index cb6c3516f..be8eab8ea 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -740,7 +740,7 @@ int JsonObject::GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &RelationalSchemaObject::GetTables() cons return tables_; } -TableInfo RelationalSchemaObject::GetTable(const std::string& tableName) const +TableInfo RelationalSchemaObject::GetTable(const std::string &tableName) const { auto it = tables_.find(tableName); if (it != tables_.end()) { @@ -481,11 +481,7 @@ int RelationalSchemaObject::ParseRelationalSchema(const JsonObject &inJsonObject if (errCode != E_OK) { return errCode; } - errCode = ParseCheckSchemaTableDefine(inJsonObject); - if (errCode != E_OK) { - return errCode; - } - return E_OK; + return ParseCheckSchemaTableDefine(inJsonObject); } int RelationalSchemaObject::ParseCheckSchemaVersionMode(const JsonObject &inJsonObject) @@ -607,7 +603,7 @@ int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject JsonObject fieldObj; errCode = inJsonObject.GetObjectByFieldPath(field.first, fieldObj); if (errCode != E_OK) { - LOGE("[RelationalSchema][Parse] Get table field object failed. %s", errCode); + LOGE("[RelationalSchema][Parse] Get table field object failed. %d", errCode); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 74ccc9423..19642b597 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -21,8 +21,6 @@ #include "auto_launch_export.h" #include "relational_store_delegate.h" -#include "irelational_store.h" -#include "relationaldb_properties.h" #include "types.h" namespace DistributedDB { @@ -44,9 +42,6 @@ public: DB_API DBStatus DeleteStore(const std::string &path); private: - void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, - const std::string &storeId, RelationalDBProperties &properties); - std::string appId_; std::string userId_; }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index a936be02a..e182aa33a 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -32,17 +32,11 @@ RelationalStoreManager::RelationalStoreManager(const std::string &appId, const s userId_(userId) {} -void RelationalStoreManager::InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, +void InitStoreProp(const std::string &storePath, const std::string &appId, const std::string &userId, const std::string &storeId, RelationalDBProperties &properties) { - // properties.SetBoolProp(RelationalDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); - properties.SetStringProp(RelationalDBProperties::APP_ID, appId_); - properties.SetStringProp(RelationalDBProperties::USER_ID, userId_); - properties.SetStringProp(RelationalDBProperties::STORE_ID, storeId); // same as dir - std::string identifier = userId_ + "-" + appId_ + "-" + storeId; - std::string hashIdentifier = DBCommon::TransferHashString(identifier); - properties.SetStringProp(RelationalDBProperties::IDENTIFIER_DATA, hashIdentifier); + properties.SetIdentifier(userId, appId, storeId); } static const int GET_CONNECT_RETRY = 3; @@ -82,7 +76,7 @@ DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const } RelationalDBProperties properties; - InitStoreProp(option, canonicalDir, storeId, properties); + InitStoreProp(canonicalDir, appId_, userId_, storeId, properties); int errCode = E_OK; auto *conn = GetOneConnectionWithRetry(properties, errCode); diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h index 1bb6f0baf..215745a8e 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h @@ -40,6 +40,9 @@ public: // Set the integer property for the name void SetIntProp(const std::string &name, int value); + // Set all indentifers + void SetIdentifier(const std::string &userId, const std::string &appId, const std::string &storeId); + static const std::string CREATE_IF_NECESSARY; static const std::string DATABASE_TYPE; static const std::string DATA_DIR; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp index 92ecca798..d6920dcc8 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp @@ -13,6 +13,7 @@ * limitations under the License. */ +#include "db_common.h" #include "db_properties.h" namespace DistributedDB { @@ -30,9 +31,8 @@ std::string DBProperties::GetStringProp(const std::string &name, const std::stri auto iter = stringProperties_.find(name); if (iter != stringProperties_.end()) { return iter->second; - } else { - return defaultValue; } + return defaultValue; } void DBProperties::SetStringProp(const std::string &name, const std::string &value) @@ -45,9 +45,8 @@ bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const auto iter = boolProperties_.find(name); if (iter != boolProperties_.end()) { return iter->second; - } else { - return defaultValue; } + return defaultValue; } void DBProperties::SetBoolProp(const std::string &name, bool value) @@ -60,13 +59,21 @@ int DBProperties::GetIntProp(const std::string &name, int defaultValue) const auto iter = intProperties_.find(name); if (iter != intProperties_.end()) { return iter->second; - } else { - return defaultValue; } + return defaultValue; } void DBProperties::SetIntProp(const std::string &name, int value) { intProperties_[name] = value; } + +void DBProperties::SetIdentifier(const std::string &userId, const std::string &appId, const std::string &storeId) +{ + SetStringProp(DBProperties::APP_ID, appId); + SetStringProp(DBProperties::USER_ID, userId); + SetStringProp(DBProperties::STORE_ID, storeId); + std::string hashIdentifier = DBCommon::TransferHashString(DBCommon::GenerateIdentifierId(storeId, appId, userId)); + SetStringProp(DBProperties::IDENTIFIER_DATA, hashIdentifier); +} } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h index 7695b7877..a9714e556 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h @@ -31,7 +31,7 @@ public: DISABLE_COPY_ASSIGN_MOVE(IRelationalStore); // Open the database. - virtual int Open(const RelationalDBProperties &kvDBProp) = 0; + virtual int Open(const RelationalDBProperties &properties) = 0; virtual void WakeUpSyncer() = 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index fe64fc818..5a8a845b1 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -59,16 +59,16 @@ RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) return connection; } -static void InitDataBaseOption(const RelationalDBProperties &kvDBProp, OpenDbProperties &option) +static void InitDataBaseOption(const RelationalDBProperties &properties, OpenDbProperties &option) { - option.uri = kvDBProp.GetStringProp(KvDBProperties::DATA_DIR, ""); - option.createIfNecessary = kvDBProp.GetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, false); + option.uri = properties.GetStringProp(DBProperties::DATA_DIR, ""); + option.createIfNecessary = properties.GetBoolProp(DBProperties::CREATE_IF_NECESSARY, false); } -int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &kvDBProp) +int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &properties) { OpenDbProperties option; - InitDataBaseOption(kvDBProp, option); + InitDataBaseOption(properties, option); StorageEngineAttr poolSize = {1, 1, 0, 16}; // at most 1 write 16 read. int errCode = sqliteStorageEngine_->InitSQLiteStorageEngine(poolSize, option); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index bd20e5c69..bb2d52f11 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -707,14 +707,6 @@ int SQLiteSingleVerRelationalStorageExecutor::CheckDBModeForRelational() LOGE("Not support journal mode %s for relational db, expect wal mode, %d", journalMode.c_str(), errCode); return -E_NOT_SUPPORT; } - - int synchronousMode; - errCode = SQLiteUtils::GetSynchronousMode(dbHandle_, synchronousMode); - if (errCode != E_OK || synchronousMode != 2) { // 2: FULL mode - LOGE("Not support synchronous mode %d for relational db, expect FULL mode, %d", synchronousMode, errCode); - return -E_NOT_SUPPORT; - } - return E_OK; } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp index b99b34f90..8ae53a1ac 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp @@ -206,7 +206,7 @@ int SQLiteSingleVerStorageEngine::AddSubscribeToMainDBInMigrate() int errCode = E_OK; auto handle = static_cast(FindExecutor(true, OperatePerm::NORMAL_PERM, errCode)); if (errCode != E_OK || handle == nullptr) { - LOGE("Get available executor for add subscribe failed. %s", errCode); + LOGE("Get available executor for add subscribe failed. %d", errCode); return errCode; } errCode = handle->StartTransaction(TransactType::IMMEDIATE); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 623335886..c839ca036 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -21,7 +21,6 @@ #include "sqlite_import.h" #include "db_types.h" -#include "iprocess_system_api_adapter.h" #include "schema_object.h" #include "types.h" #ifdef RELATIONAL_STORE diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp index 50496bb84..f8c8df347 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp @@ -64,7 +64,7 @@ namespace { std::string GetKvStoreDirectory(const std::string &userId, const std::string &appId, const std::string &storeId) { - string identifier = userId + "-" + appId + "-" + storeId; + string identifier = DBCommon::GenerateIdentifierId(storeId, appId, userId); string hashIdentifierName = DBCommon::TransferHashString(identifier); string identifierName = DBCommon::TransferStringToHex(hashIdentifierName); string filePath = g_testDir + "/" + identifierName + "/" + DBConstant::SINGLE_SUB_DIR + "/main/"; -- Gitee From 5ed241b2331d66685f49538b4622fc1a07264dd8 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Thu, 30 Dec 2021 17:16:49 +0800 Subject: [PATCH 21/53] Fix reviewbot. Signed-off-by: lidwchn --- .../libs/distributeddb/common/include/db_types.h | 11 +++++++++++ .../src/relational/relational_sync_able_storage.h | 2 +- .../storage/include/isync_interface.h | 11 ----------- .../distributeddb/storage/src/data_transformer.cpp | 7 ++++--- .../storage/src/relational_sync_able_storage.cpp | 14 +++++++------- .../sqlite_single_ver_relational_continue_token.h | 1 - 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_types.h b/services/distributeddataservice/libs/distributeddb/common/include/db_types.h index 697600aa8..94f6cbd1c 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_types.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_types.h @@ -123,5 +123,16 @@ enum SingleVerConflictResolvePolicy { DEFAULT_LAST_WIN = 0, DENY_OTHER_DEV_AMEND_CUR_DEV_DATA = 1, }; + +struct SyncTimeRange { + TimeStamp beginTime = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp endTime = static_cast(INT64_MAX); + TimeStamp deleteEndTime = static_cast(INT64_MAX); + bool IsValid() const + { + return (beginTime <= endTime && deleteBeginTime <= deleteEndTime); + } +}; } // namespace DistributedDB #endif // DISTRIBUTEDDB_TYPES_H diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index 05bb9b1bf..e9e6562b4 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -109,7 +109,7 @@ private: int SetMaxTimeStamp(TimeStamp timestamp); // get - int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerRelationalContinueToken *&continueStmtToken, + int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerRelationalContinueToken *&token, const DataSizeSpecInfo &dataSizeInfo) const; // put diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h index 375823e39..a6925c11d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h @@ -22,17 +22,6 @@ #include "kvdb_properties.h" namespace DistributedDB { -struct SyncTimeRange { - TimeStamp beginTime = 0; - TimeStamp deleteBeginTime = 0; - TimeStamp endTime = static_cast(INT64_MAX); - TimeStamp deleteEndTime = static_cast(INT64_MAX); - bool IsValid() const - { - return (beginTime <= endTime && deleteBeginTime <= deleteEndTime); - } -}; - class ISyncInterface { public: enum { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp index c4e020261..b004d4d60 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -118,17 +118,18 @@ uint32_t DataTransformer::CalDataValueLength(const DataValue &dataValue) return 0u; } uint32_t length = 0; - std::string str; switch (dataValue.GetType()) { case StorageType::STORAGE_TYPE_BLOB: (void)dataValue.GetBlobLength(length); length = Parcel::GetEightByteAlign(length); length += Parcel::GetUInt32Len(); // record data length break; - case StorageType::STORAGE_TYPE_TEXT: + case StorageType::STORAGE_TYPE_TEXT: { + std::string str; (void)dataValue.GetText(str); length = Parcel::GetStringLen(str); break; + } default: break; } @@ -245,7 +246,7 @@ int SerializeBlobValue(const DataValue &dataValue, Parcel &parcel) { Blob val; (void)dataValue.GetBlob(val); - const uint32_t &size = val.GetSize(); + uint32_t size = val.GetSize(); if (size == 0) { return SerializeNullValue(dataValue, parcel); } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index 3f18bdd04..2cc4aab84 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -256,11 +256,11 @@ static void ProcessContinueTokenForQuerySync(const std::vector &dataIt } /** - * Caller must ensure that parameter continueStmtToken is valid. + * Caller must ensure that parameter token is valid. * If error happened, token will be deleted here. */ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &dataItems, - SQLiteSingleVerRelationalContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const + SQLiteSingleVerRelationalContinueToken *&token, const DataSizeSpecInfo &dataSizeInfo) const { if (storageEngine_ == nullptr) { return -E_INVALID_DB; @@ -273,7 +273,7 @@ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &da goto ERROR; } - errCode = handle->SetTableInfo(continueStmtToken->GetQuery()); + errCode = handle->SetTableInfo(token->GetQuery()); if (errCode != E_OK) { goto ERROR; } @@ -282,11 +282,11 @@ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &da errCode = handle->GetSyncDataByQuery(dataItems, Parcel::GetAppendedLen(), dataSizeInfo, - std::bind(&SQLiteSingleVerRelationalContinueToken::GetStatement, *continueStmtToken, + std::bind(&SQLiteSingleVerRelationalContinueToken::GetStatement, *token, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); if (errCode == -E_FINISHED) { - continueStmtToken->FinishGetData(); - errCode = continueStmtToken->IsGetAllDataFinished() ? E_OK : -E_UNFINISHED; + token->FinishGetData(); + errCode = token->IsGetAllDataFinished() ? E_OK : -E_UNFINISHED; } } while (errCode == -E_UNFINISHED && CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())); @@ -295,7 +295,7 @@ ERROR: if (errCode != -E_UNFINISHED && errCode != E_OK) { // Error happened. dataItems.clear(); } - ProcessContinueTokenForQuerySync(dataItems, errCode, continueStmtToken); + ProcessContinueTokenForQuerySync(dataItems, errCode, token); ReleaseHandle(handle); return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h index a6542b488..912aa4c96 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h @@ -20,7 +20,6 @@ #include "db_types.h" #include "query_object.h" -#include "single_ver_kvdb_sync_interface.h" namespace DistributedDB { class SQLiteSingleVerRelationalContinueToken { -- Gitee From 6d6a8a6cddd97d8c273062f6ebc69fe25292321c Mon Sep 17 00:00:00 2001 From: zwtmichael Date: Tue, 28 Dec 2021 11:58:04 +0800 Subject: [PATCH 22/53] slide window refactoring Signed-off-by: zwtmichael --- .../libs/distributeddb/BUILD.gn | 4 +- .../src/single_ver_data_message_schedule.cpp | 331 +++++++++++++ ...r.h => single_ver_data_message_schedule.h} | 85 ++-- .../syncer/src/single_ver_data_sync.cpp | 279 ++++++++++- .../syncer/src/single_ver_data_sync.h | 67 ++- ...ngle_ver_data_sync_with_sliding_window.cpp | 66 --- ...single_ver_data_sync_with_sliding_window.h | 45 -- .../src/single_ver_sync_state_machine.cpp | 352 +++----------- .../src/single_ver_sync_state_machine.h | 32 +- .../src/single_ver_sync_task_context.cpp | 28 +- .../syncer/src/single_ver_sync_task_context.h | 19 - .../syncer/src/sliding_window_receiver.cpp | 278 ----------- .../syncer/src/sliding_window_sender.cpp | 295 ----------- .../syncer/src/sliding_window_sender.h | 77 --- .../syncer/src/sync_state_machine.cpp | 11 +- .../syncer/src/sync_task_context.cpp | 2 +- .../libs/distributeddb/test/BUILD.gn | 10 +- ...ributeddb_single_ver_msg_schedule_test.cpp | 456 ++++++++++++++++++ .../syncer/distributeddb_sync_module_test.cpp | 130 ----- .../common/syncer/mock_single_ver_data_sync.h | 28 -- .../mock_single_ver_sync_task_context.h | 28 -- 21 files changed, 1248 insertions(+), 1375 deletions(-) create mode 100644 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp rename services/distributeddataservice/libs/distributeddb/syncer/src/{sliding_window_receiver.h => single_ver_data_message_schedule.h} (33%) mode change 100755 => 100644 delete mode 100755 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp delete mode 100755 services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h delete mode 100755 services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp delete mode 100755 services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp delete mode 100755 services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h create mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp delete mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp delete mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h delete mode 100644 services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index 29f9c8c04..22cc35453 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -213,9 +213,9 @@ ohos_shared_library("distributeddb") { "syncer/src/multi_ver_sync_task_context.cpp", "syncer/src/multi_ver_syncer.cpp", "syncer/src/query_sync_water_mark_helper.cpp", + "syncer/src/single_ver_data_message_schedule.cpp" "syncer/src/single_ver_data_packet.cpp", "syncer/src/single_ver_data_sync.cpp", - "syncer/src/single_ver_data_sync_with_sliding_window.cpp", "syncer/src/single_ver_kv_syncer.cpp", "syncer/src/single_ver_relational_syncer.cpp", "syncer/src/single_ver_serialize_manager.cpp", @@ -224,8 +224,6 @@ ohos_shared_library("distributeddb") { "syncer/src/single_ver_sync_target.cpp", "syncer/src/single_ver_sync_task_context.cpp", "syncer/src/single_ver_syncer.cpp", - "syncer/src/sliding_window_receiver.cpp", - "syncer/src/sliding_window_sender.cpp", "syncer/src/subscribe_manager.cpp", "syncer/src/sync_engine.cpp", "syncer/src/sync_operation.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp new file mode 100644 index 000000000..ede29bb5a --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "log_print.h" +#include "version.h" +#include "single_ver_data_sync.h" +#include "single_ver_data_message_schedule.h" + +namespace DistributedDB { +SingleVerDataMessageSchedule::~SingleVerDataMessageSchedule() +{ + LOGD("~SingleVerDataMessageSchedule"); + ClearMsg(); +} + +void SingleVerDataMessageSchedule::Initialize(const std::string &label, const std::string &deviceId) +{ + label_ = label; + deviceId_ = deviceId; +} + +void SingleVerDataMessageSchedule::PutMsg(Message *inMsg) +{ + if (inMsg == nullptr) { + return; + } + std::lock_guard lock(queueLock_); + msgQueue_.push(inMsg); + isNeedReload_ = true; +} + +bool SingleVerDataMessageSchedule::IsNeedReloadQueue() +{ + std::lock_guard lock(queueLock_); + return isNeedReload_; +} + +Message *SingleVerDataMessageSchedule::MoveNextMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, + bool &isNeedContinue) +{ + uint32_t remoteVersion = context->GetRemoteSoftwareVersion(); + if (remoteVersion < SOFTWARE_VERSION_RELEASE_3_0) { + // just get last msg when remote version is < 103 or >=103 but just open db now + return GetLastMsgFromQueue(); + } + { + std::lock_guard lock(workingLock_); + if (isWorking_) { + isNeedContinue = false; + return nullptr; + } + isWorking_ = true; + } + ResetTimer(context); + UpdateMsgMap(); + Message *msg = GetMsgFromMap(isNeedHandle); + isNeedContinue = true; + if (msg == nullptr) { + std::lock_guard lock(workingLock_); + isWorking_ = false; + return nullptr; + } + return msg; +} + +void SingleVerDataMessageSchedule::ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, + const Message *inMsg) +{ + if (isNeedHandleStatus) { + const DataRequestPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + LOGE("[DataMsgSchedule] packet is nullptr"); + return; + } + uint64_t curPacketId = packet->GetPacketId(); + { + std::lock_guard lock(lock_); + finishedPacketId_ = curPacketId; + if (isNeedClearMap) { + ClearMsgMapWithNoLock(); + expectedSequenceId_ = 1; + } else { + LOGI("[DataMsgSchedule] DealMsg seqId=%u finishedPacketId=%llu ok,label=%s,deviceId=%s{private}", + expectedSequenceId_, finishedPacketId_, label_.c_str(), deviceId_.c_str()); + expectedSequenceId_++; + } + } + } + std::lock_guard lock(workingLock_); + isWorking_ = false; +} + +void SingleVerDataMessageSchedule::ClearMsg() +{ + StopTimer(); + ClearMsgQueue(); + ClearMsgMap(); +} + +void SingleVerDataMessageSchedule::UpdateMsgMap() +{ + std::queue msgTmpQueue; + { + std::lock_guard lock(queueLock_); + while (!msgQueue_.empty()) { + msgTmpQueue.push(msgQueue_.front()); + msgQueue_.pop(); + } + isNeedReload_ = false; + } + UpdateMsgMapInner(msgTmpQueue); +} + +void SingleVerDataMessageSchedule::UpdateMsgMapInner(std::queue &msgTmpQueue) +{ + // update msg map + std::lock_guard lock(lock_); + while (!msgTmpQueue.empty()) { + Message *msg = msgTmpQueue.front(); + msgTmpQueue.pop(); + if (msg == nullptr) { + continue; + } + const DataRequestPacket *packet = msg->GetObject(); + if (packet == nullptr) { + delete msg; + continue; + } + uint32_t sessionId = msg->GetSessionId(); + uint32_t sequenceId = msg->GetSequenceId(); + uint64_t packetId = packet->GetPacketId(); + if (prevSessionId_ != 0 && sessionId == prevSessionId_) { + LOGD("[DataMsgSchedule] recv prev sessionId msg,drop msg,label=%s,deviceId=%s{private}", + label_.c_str(), deviceId_.c_str()); + delete msg; + continue; + } + if (sessionId != currentSessionId_) { + // make sure all msg sessionId is same in msgMap + ClearMsgMapWithNoLock(); + prevSessionId_ = currentSessionId_; + currentSessionId_ = sessionId; + finishedPacketId_ = 0; + expectedSequenceId_ = 1; + } + if (messageMap_.count(sequenceId) > 0) { + const auto *cachePacket = messageMap_[sequenceId]->GetObject(); + LOGD("[DataMsgSchedule] msg packetId=%llu,cachePacketId=%llu,label=%s,deviceId=%s", packetId, + cachePacket->GetPacketId(), label_.c_str(), deviceId_.c_str()); + if (packetId != 0 && packetId < cachePacket->GetPacketId()) { + delete msg; + continue; + } + delete messageMap_[sequenceId]; + messageMap_[sequenceId] = nullptr; + } + messageMap_[sequenceId] = msg; + LOGD("[DataMsgSchedule] put into msgMap seqId=%llu,packetId=%llu,label=%s,deviceId=%s", sequenceId, + packetId, label_.c_str(), deviceId_.c_str()); + } +} + +Message *SingleVerDataMessageSchedule::GetMsgFromMap(bool &isNeedHandle) +{ + isNeedHandle = true; + std::lock_guard lock(lock_); + while (!messageMap_.empty()) { + auto iter = messageMap_.begin(); + Message *msg = iter->second; + messageMap_.erase(iter); + const DataRequestPacket *packet = msg->GetObject(); + if (packet == nullptr) { + LOGE("[DataMsgSchedule] expected error"); + delete msg; + continue; + } + uint32_t sequenceId = msg->GetSequenceId(); + uint64_t packetId = packet->GetPacketId(); + if (sequenceId < expectedSequenceId_) { + uint64_t revisePacketId = finishedPacketId_ - (expectedSequenceId_ - 1 - sequenceId); + LOGI("[DataMsgSchedule] msg seqId=%llu less than exSeqId=%llu,pacId=%llu,revisePacId=%llu,label=%s,dev=%s", + sequenceId, expectedSequenceId_, packetId, revisePacketId, label_.c_str(), deviceId_.c_str()); + if (packetId < revisePacketId) { + delete msg; + continue; + } + // means already handle the msg, and just send E_OK ack in dataSync + isNeedHandle = false; + return msg; + } + if (sequenceId == expectedSequenceId_) { + if (packetId < finishedPacketId_) { + LOGI("[DataMsgSchedule] drop msg seqId=%llu,packetId=%llu,label=%s,deviceId=%s", sequenceId, + packetId, label_.c_str(), deviceId_.c_str()); + delete msg; + continue; + } + // if packetId == finishedPacketId_ need handle + // it will happened while watermark/need_abilitySync when last ack is missing + return msg; + } + // sequenceId > expectedSequenceId_, not need handle, put into map again + messageMap_[sequenceId] = msg; + return nullptr; + } + return nullptr; +} + +Message *SingleVerDataMessageSchedule::GetLastMsgFromQueue() +{ + std::lock_guard lock(queueLock_); + isNeedReload_ = false; + while (!msgQueue_.empty()) { + Message *msg = msgQueue_.front(); + msgQueue_.pop(); + if (msgQueue_.empty()) { // means last msg + return msg; + } + delete msg; + } + return nullptr; +} + +void SingleVerDataMessageSchedule::ClearMsgMap() +{ + std::lock_guard lock(lock_); + ClearMsgMapWithNoLock(); +} + +void SingleVerDataMessageSchedule::ClearMsgMapWithNoLock() +{ + LOGD("[DataMsgSchedule] begin to ClearMsgMapWithNoLock"); + for (auto &iter : messageMap_) { + delete iter.second; + iter.second = nullptr; + } + messageMap_.clear(); +} + +void SingleVerDataMessageSchedule::ClearMsgQueue() +{ + std::lock_guard lock(queueLock_); + while (!msgQueue_.empty()) { + Message *msg = msgQueue_.front(); + msgQueue_.pop(); + delete msg; + } +} + +void SingleVerDataMessageSchedule::StartTimer(SingleVerSyncTaskContext *context) +{ + std::lock_guard lock(lock_); + TimerId timerId = 0; + RefObject::IncObjRef(context); + TimerAction timeOutCallback = std::bind(&SingleVerDataMessageSchedule::TimeOut, this, std::placeholders::_1); + int errCode = RuntimeContext::GetInstance()->SetTimer(IDLE_TIME_OUT, timeOutCallback, + [context]() { + int errCode = RuntimeContext::GetInstance()->ScheduleTask([context]() { + RefObject::DecObjRef(context); + }); + if (errCode != E_OK) { + LOGE("[DataMsgSchedule] timer finalizer ScheduleTask,errCode=%d", errCode); + } + }, timerId); + if (errCode != E_OK) { + RefObject::DecObjRef(context); + LOGE("[DataMsgSchedule] timer ScheduleTask, errCode=%d", errCode); + return; + } + timerId_ = timerId; + LOGD("[DataMsgSchedule] StartTimer,TimerId=%llu", timerId_); +} + +void SingleVerDataMessageSchedule::StopTimer() +{ + TimerId timerId; + { + std::lock_guard lock(lock_); + LOGD("[DataMsgSchedule] StopTimer,remove TimerId=%llu", timerId_); + if (timerId_ == 0) { + return; + } + timerId = timerId_; + timerId_ = 0; + } + RuntimeContext::GetInstance()->RemoveTimer(timerId); +} + +void SingleVerDataMessageSchedule::ResetTimer(SingleVerSyncTaskContext *context) +{ + StopTimer(); + StartTimer(context); +} + +int SingleVerDataMessageSchedule::TimeOut(TimerId timerId) +{ + if (IsNeedReloadQueue()) { + LOGI("[DataMsgSchedule] new msg exists, no need to timeout handle"); + return E_OK; + } + { + std::lock_guard lock(workingLock_); + if (isWorking_) { + LOGI("[DataMsgSchedule] other thread is handle msg, no need to timeout handle"); + return E_OK; + } + } + { + std::lock_guard lock(lock_); + LOGI("[DataMsgSchedule] timeout handling, stop timerId_[%llu]", timerId, timerId_); + if (timerId == timerId_) { + ClearMsgMapWithNoLock(); + timerId_ = 0; + } + } + RuntimeContext::GetInstance()->RemoveTimer(timerId); + return E_OK; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h old mode 100755 new mode 100644 similarity index 33% rename from services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h index 461ac6465..e87362634 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h @@ -13,57 +13,62 @@ * limitations under the License. */ -#ifndef SLIDING_WINDOW_RECEIVER_H -#define SLIDING_WINDOW_RECEIVER_H - #include +#include +#include +#include #include "message.h" -#include "single_ver_data_sync.h" +#include "runtime_context.h" #include "single_ver_sync_task_context.h" +#ifndef SINGLE_VER_DATA_MESSAGE_SCHEDULE_H +#define SINGLE_VER_DATA_MESSAGE_SCHEDULE_H namespace DistributedDB { -class SlidingWindowReceiver { +class SingleVerDataMessageSchedule { public: - SlidingWindowReceiver() = default; - ~SlidingWindowReceiver(); - DISABLE_COPY_ASSIGN_MOVE(SlidingWindowReceiver); - - int Initialize(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int Receive(Message *inMsg); - void Clear(); + SingleVerDataMessageSchedule() = default; + ~SingleVerDataMessageSchedule(); + void Initialize(const std::string &label, const std::string &deviceId); + void PutMsg(Message *inMsg); + bool IsNeedReloadQueue(); + Message *MoveNextMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, bool &isNeedContinue); + void ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, const Message *inMsg); + void ClearMsg(); private: - int PutMsg(Message *inMsg); - void DealMsg(); - int TimeOut(TimerId timerId); - void StartTimer(); + void UpdateMsgMap(); + void UpdateMsgMapInner(std::queue &msgTmpQueue); + Message *GetMsgFromMap(bool &isNeedHandle); + Message *GetLastMsgFromQueue(); + void ClearMsgMap(); + void ClearMsgMapWithNoLock(); + void ClearMsgQueue(); + void StartTimer(SingleVerSyncTaskContext *context); void StopTimer(); - static void StopTimer(TimerId timerId); - void ClearMap(); - void ResetInfo(); - int ErrHandle(uint32_t sequenceId, uint64_t packetId); - void SetEndField(bool isLastSequence, uint32_t sequenceId); + void ResetTimer(SingleVerSyncTaskContext *context); + // when timeout queue size is 0 because thread can move queue msg to map if isNeedReload which is + // actived when queue has new msg is true + // so only need clear map msg + int TimeOut(TimerId timerId); + static constexpr int IDLE_TIME_OUT = 5 * 60 * 1000; // 5min + std::mutex queueLock_; + std::queue msgQueue_; + bool isNeedReload_ = false; + // only one thread is deal msg + std::mutex workingLock_; + bool isWorking_ = false; + // first:sequenceId second:Message*, deal msg from low sequenceId to high sequenceId std::mutex lock_; - std::condition_variable workingTaskcv_; - // 0 is default invalid. - uint32_t sessionId_ = 0; - // first:sequenceId second:Message*, message data not deal. std::map messageMap_; - // 0 is has finished nothing; e.g. 3 is sequenceId 1 2 3 has finished, sequenceId 4 has not finished. - uint32_t hasFinishedMaxId_ = 0; - uint64_t hasFinishedPacketId_ = 0; - // 0 is idle - uint32_t workingId_ = 0; - // 0 is has not received the end packet now. - uint32_t endId_ = 0; - bool isWaterMarkErrHappened_ = false; - // timeout for idle wait packet + uint32_t prevSessionId_ = 0; // drop the msg if msg sessionId is prev sessionId. + uint32_t currentSessionId_ = 0; + uint64_t finishedPacketId_ = 0; // next msg packetId should larger than it + uint32_t expectedSequenceId_ = 0; // handle next msg which sequenceId is equal to it TimerId timerId_ = 0; - static constexpr int IDLE_TIME_OUT = 5 * 60 * 1000; // 5min - SingleVerSyncTaskContext *context_ = nullptr; - std::shared_ptr dataSync_; -}; -} // namespace DistributedDB -#endif // SLIDING_WINDOW_RECEIVER_H + std::string label_; + std::string deviceId_; +}; +} +#endif // SINGLE_VER_DATA_MESSAGE_SCHEDULE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index 46cd9692e..eca7485c8 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -74,9 +74,152 @@ int SingleVerDataSync::Initialize(ISyncInterface *inStorage, ICommunicator *inCo label.resize(3); // only show 3 Bytes enough label_ = DBCommon::VectorToHexString(label); deviceId_ = deviceId; + msgSchedule_.Initialize(label_, deviceId_); return E_OK; } +int SingleVerDataSync::SyncStart(int mode, SingleVerSyncTaskContext *context) +{ + std::lock_guard lock(lock_); + if (sessionId_ != 0) { // auto sync timeout resend + return ReSendData(context); + } + ResetSyncStatus(mode, context); + LOGI("[DataSync] SendStart,mode=%d,label=%s,device=%s", mode_, label_.c_str(), STR_MASK(deviceId_)); + int tmpMode = SyncOperation::TransferSyncMode(mode); + int errCode = E_OK; + if (tmpMode == SyncModeType::PUSH) { + errCode = PushStart(context); + } else if (tmpMode == SyncModeType::PUSH_AND_PULL) { + errCode = PushPullStart(context); + } else if (tmpMode == SyncModeType::PULL) { + errCode = PullRequestStart(context); + } else { + errCode = PullResponseStart(context); + } + if (context->IsSkipTimeoutError(errCode)) { + // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. + // just return to avoid higher pressure for send. + return E_OK; + } + if (errCode != E_OK) { + LOGE("[DataSync] SendStart errCode=%d", errCode); + return errCode; + } + if (tmpMode == SyncModeType::PUSH_AND_PULL && context->GetTaskErrCode() == -E_EKEYREVOKED) { + LOGE("wait for recv finished for push and pull mode"); + return -E_EKEYREVOKED; + } + return InnerSyncStart(context); +} + +int SingleVerDataSync::InnerSyncStart(SingleVerSyncTaskContext *context) +{ + while (true) { + if (windowSize_ <= 0 || isAllDataHasSent_) { + LOGD("[DataSync] InnerDataSync winSize=%d,isAllSent=%d,label=%s,device=%s", windowSize_, isAllDataHasSent_, + label_.c_str(), STR_MASK(deviceId_)); + return E_OK; + } + int mode = SyncOperation::TransferSyncMode(mode_); + if (mode == SyncModeType::PULL) { + LOGE("[DataSync] unexpected error"); + return -E_INVALID_ARGS; + } + int errCode; + context->IncSequenceId(); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { + errCode = PushStart(context); + } else { + errCode = PullResponseStart(context); + } + if ((mode == SyncModeType::PUSH_AND_PULL) && errCode == -E_EKEYREVOKED) { + LOGE("[DataSync] wait for recv finished,label=%s,device=%s", label_.c_str(), STR_MASK(deviceId_)); + isAllDataHasSent_ = true; + return -E_EKEYREVOKED; + } + if (context->IsSkipTimeoutError(errCode)) { + // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. + // just return to avoid higher pressure for send. + return E_OK; + } + if (errCode != E_OK) { + LOGE("[DataSync] InnerSend errCode=%d", errCode); + return errCode; + } + } + return E_OK; +} + +void SingleVerDataSync::InnerClearSyncStatus() +{ + sessionId_ = 0; + reSendMap_.clear(); + windowSize_ = 0; + maxSequenceIdHasSent_ = 0; + isAllDataHasSent_ = false; +} + +int SingleVerDataSync::TryContinueSync(SingleVerSyncTaskContext *context, const Message *message) +{ + if (message == nullptr) { + LOGE("[DataSync] AckRecv message nullptr"); + return -E_INVALID_ARGS; + } + const DataAckPacket *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value + uint32_t sessionId = message->GetSessionId(); + uint32_t sequenceId = message->GetSequenceId(); + + std::lock_guard lock(lock_); + LOGI("[DataSync] recv ack seqId=%d,packetId=%llu,winSize=%d,label=%s,dev=%s", sequenceId, packetId, windowSize_, + label_.c_str(), STR_MASK(deviceId_)); + if (sessionId != sessionId_) { + LOGI("[DataSync] ignore ack,sessionId is different"); + return E_OK; + } + if (reSendMap_.count(sequenceId) != 0) { + reSendMap_.erase(sequenceId); + windowSize_++; + } else { + LOGI("[DataSync] ack seqId not in map"); + return E_OK; + } + if (!isAllDataHasSent_) { + return InnerSyncStart(context); + } else if (reSendMap_.size() == 0) { + context->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); + InnerClearSyncStatus(); + return -E_FINISHED; + } + return E_OK; +} + +void SingleVerDataSync::ClearSyncStatus() +{ + std::lock_guard lock(lock_); + InnerClearSyncStatus(); +} + +int SingleVerDataSync::ReSendData(SingleVerSyncTaskContext *context) +{ + if (reSendMap_.empty()) { + LOGI("[DataSync] ReSend map empty"); + return -E_INTERNAL_ERROR; + } + uint32_t sequenceId = reSendMap_.begin()->first; + ReSendInfo reSendInfo = reSendMap_.begin()->second; + LOGI("[DataSync] ReSend mode=%d,start=%llu,end=%llu,delStart=%llu,delEnd=%llu,seqId=%d,packetId=%llu,windowsize=%d," + "label=%s,deviceId=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, sequenceId, reSendInfo.packetId, windowSize_, label_.c_str(), STR_MASK(deviceId_)); + DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, + reSendInfo.deleteBeginTime, reSendInfo.deleteEndTime, reSendInfo.packetId}; + return ReSend(context, dataReSendInfo); +} + std::string SingleVerDataSync::GetLocalDeviceName() { std::string deviceInfo; @@ -252,6 +395,26 @@ int SingleVerDataSync::SaveData(const SingleVerSyncTaskContext *context, const s return errCode; } +void SingleVerDataSync::ResetSyncStatus(int inMode, SingleVerSyncTaskContext *context) +{ + mode_ = inMode; + maxSequenceIdHasSent_ = 0; + isAllDataHasSent_ = false; + context->ReSetSequenceId(); + reSendMap_.clear(); + if (context->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { + windowSize_ = LOW_VERSION_WINDOW_SIZE; + } else { + windowSize_ = HIGH_VERSION_WINDOW_SIZE; + } + int mode = SyncOperation::TransferSyncMode(inMode); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL) { + sessionId_ = context->GetRequestSessionId(); + } else { + sessionId_ = context->GetResponseSessionId(); + } +} + TimeStamp SingleVerDataSync::GetMaxSendDataTime(const std::vector &inData) { TimeStamp stamp = 0; @@ -574,6 +737,38 @@ void SingleVerDataSync::TransSendDataItemToLocal(const SingleVerSyncTaskContext } } +void SingleVerDataSync::SetSessionEndTimeStamp(TimeStamp end) +{ + sessionEndTimeStamp_ = end; +} + +TimeStamp SingleVerDataSync::GetSessionEndTimeStamp() const +{ + return sessionEndTimeStamp_; +} + +void SingleVerDataSync::UpdateSendInfo(SyncTimeRange dataTimeRange, SingleVerSyncTaskContext *context) +{ + ReSendInfo reSendInfo; + reSendInfo.start = dataTimeRange.beginTime; + reSendInfo.end = dataTimeRange.endTime; + reSendInfo.deleteBeginTime = dataTimeRange.deleteBeginTime; + reSendInfo.deleteEndTime = dataTimeRange.deleteEndTime; + reSendInfo.packetId = context->GetPacketId(); + maxSequenceIdHasSent_++; + reSendMap_[maxSequenceIdHasSent_] = reSendInfo; + windowSize_--; + ContinueToken token; + context->GetContinueToken(token); + if (token == nullptr) { + isAllDataHasSent_ = true; + } + LOGI("[DataSync] mode=%d,start=%llu,end=%llu,deleteStart=%llu,deleteEnd=%llu,seqId=%d,packetId=%llu,window_size=%d," + "isAllSend=%d,label=%s,device=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, maxSequenceIdHasSent_, reSendInfo.packetId, windowSize_,isAllDataHasSent_, + label_.c_str(), STR_MASK(deviceId_)); +} + void SingleVerDataSync::FillDataRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, SyncEntry &syncData, int sendCode, int mode) { @@ -654,9 +849,8 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; UpdateWaterMark isUpdateWaterMark; SyncTimeRange dataTime = GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); - context->SetSequenceStartAndEndTimeStamp(dataTime); if (errCode == E_OK) { - context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); + SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } FillDataRequestPacket(packet, context, syncData, errCode, mode); errCode = SendDataPacket(curType, packet, context); @@ -664,6 +858,9 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) if (performance != nullptr) { performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_MACHINE_START_TO_PUSH_SEND); } + if (errCode == E_OK || errCode == -E_TIMEOUT) { + UpdateSendInfo(dataTime, context); + } if (errCode == E_OK) { if (curType == SyncType::QUERY_SYNC_TYPE && (context->GetQuery().HasLimit() || context->GetQuery().HasOrderBy())) { @@ -723,7 +920,6 @@ int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); WaterMark endMark = context->GetEndMark(); SyncTimeRange dataTime = {localMark, deleteMark, localMark, deleteMark}; - context->SetSequenceStartAndEndTimeStamp(dataTime); packet->SetBasicInfo(E_OK, version, context->GetMode()); packet->SetWaterMark(localMark, peerMark, deleteMark); @@ -736,6 +932,7 @@ int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) LOGD("[DataSync][Pull] curType=%d,local=%llu,del=%llu,end=%llu,peer=%llu,label=%s,dev=%s", syncType, localMark, deleteMark, peerMark, endMark, label_.c_str(), STR_MASK(GetDeviceId())); + UpdateSendInfo(dataTime, context); return SendDataPacket(syncType, packet, context); } @@ -763,13 +960,14 @@ int SingleVerDataSync::PullResponseStart(SingleVerSyncTaskContext *context) } SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; UpdateWaterMark isUpdateWaterMark; - SyncTimeRange dataTime = - GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); - context->SetSequenceStartAndEndTimeStamp(dataTime); + SyncTimeRange dataTime = GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); if (errCode == E_OK) { - context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); + SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } errCode = SendPullResponseDataPkt(ackCode, syncData, context); + if (errCode == E_OK || errCode == -E_TIMEOUT) { + UpdateSendInfo(dataTime, context); + } if (errCode == E_OK) { if (curType == SyncType::QUERY_SYNC_TYPE && (context->GetQuery().HasLimit() || context->GetQuery().HasOrderBy())) { @@ -1016,6 +1214,11 @@ int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, SyncEntry &syncOutDa return errCode; } +void SingleVerDataSync::SendFinishedDataAck(SingleVerSyncTaskContext *context, const Message *message) +{ + SendDataAck(context, message, E_OK, 0); +} + int SingleVerDataSync::SendDataAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, WaterMark maxSendDataTime) { @@ -1048,6 +1251,32 @@ int SingleVerDataSync::SendDataAck(SingleVerSyncTaskContext *context, const Mess return errCode; } +bool SingleVerDataSync::AckPacketIdCheck(const Message *message) +{ + if (message == nullptr) { + LOGE("[DataSync] AckRecv message nullptr"); + return false; + } + if (message->GetMessageType() == TYPE_NOTIFY || message->IsFeedbackError()) { + return false; + } + const DataAckPacket *packet = message->GetObject(); + if (packet == nullptr) { + return false; + } + uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value + std::lock_guard lock(lock_); + uint32_t sequenceId = message->GetSequenceId(); + if (reSendMap_.count(sequenceId) != 0) { + uint64_t originalPacketId = reSendMap_[sequenceId].packetId; + if (DataAckPacket::IsPacketIdValid(packetId) && packetId != originalPacketId) { + LOGE("[DataSync] packetId[%llu] is not match with original[%llu]", packetId, originalPacketId); + return false; + } + } + return true; +} + int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message *message) { int errCode = AckMsgErrnoCheck(context, message); @@ -1067,8 +1296,7 @@ int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message } if (recvCode == -E_NEED_ABILITY_SYNC || recvCode == -E_NOT_PERMIT) { - // after set sliding window sender err, we can ReleaseContinueToken, avoid crash - context->SetSlidingWindowSenderErr(true); + // we should ReleaseContinueToken, avoid crash LOGI("[DataSync][AckRecv] Data sync abort,recvCode =%d,label =%s,dev=%s", recvCode, label_.c_str(), STR_MASK(GetDeviceId())); context->ReleaseContinueToken(); @@ -1158,9 +1386,6 @@ void SingleVerDataSync::GetPullEndWatermark(const SingleVerSyncTaskContext *cont int SingleVerDataSync::DealWaterMarkException(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, const std::vector &reserved) { - // after set sliding window sender err, we can SaveLocalWaterMark, avoid sliding window sender re save a wrong - // waterMark again. - context->SetSlidingWindowSenderErr(true); WaterMark deletedWaterMark = 0; SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; if (curType == SyncType::QUERY_SYNC_TYPE) { @@ -1660,12 +1885,12 @@ void SingleVerDataSync::FillRequestReSendPacket(const SingleVerSyncTaskContext * // transer reSend mode, RESPONSE_PULL transfer to push or query push // PUSH_AND_PULL mode which sequenceId lager than first transfer to push or query push int reSendMode = GetReSendMode(context->GetMode(), reSendInfo.sequenceId, curType); - if (context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) || + if (GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) || SyncOperation::TransferSyncMode(context->GetMode()) == SyncModeType::PULL) { LOGI("[DataSync][ReSend] set lastid,label=%s,dev=%s", label_.c_str(), STR_MASK(GetDeviceId())); packet->SetLastSequence(); } - if (sendCode == E_OK && context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) && + if (sendCode == E_OK && GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) && context->GetMode() == SyncModeType::RESPONSE_PULL) { sendCode = SEND_FINISHED; } @@ -2148,4 +2373,30 @@ std::string SingleVerDataSync::GetDeleteSyncId(const SingleVerSyncTaskContext *c #endif return id; } + +void SingleVerDataSync::PutDataMsg(Message *message) +{ + return msgSchedule_.PutMsg(message); +} + +Message *SingleVerDataSync::MoveNextDataMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, + bool &isNeedContinue) +{ + return msgSchedule_.MoveNextMsg(context, isNeedHandle, isNeedContinue); +} + +bool SingleVerDataSync::IsNeedReloadQueue() +{ + return msgSchedule_.IsNeedReloadQueue(); +} + +void SingleVerDataSync::ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, const Message *message) +{ + msgSchedule_.ScheduleInfoHandle(isNeedHandleStatus, isNeedClearMap, message); +} + +void SingleVerDataSync::ClearDataMsg() +{ + msgSchedule_.ClearMsg(); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h index bcb2b0b97..8f0e4b023 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h @@ -20,6 +20,7 @@ #include "isync_interface.h" #include "meta_data.h" #include "parcel.h" +#include "single_ver_data_message_schedule.h" #include "single_ver_data_packet.h" #include "single_ver_kvdb_sync_interface.h" #include "single_ver_sync_task_context.h" @@ -29,6 +30,15 @@ namespace DistributedDB { using SendDataItem = SingleVerKvEntry *; +struct ReSendInfo { + TimeStamp start = 0; + TimeStamp end = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp deleteEndTime = 0; + // packetId is used for matched ackpacket packetId which saved in ackPacket.reserve + // if equaled, means need to handle the ack, or drop. it is always increased + uint64_t packetId = 0; +}; struct DataSyncReSendInfo { uint32_t sessionId = 0; @@ -54,6 +64,12 @@ public: int Initialize(ISyncInterface *inStorage, ICommunicator *inCommunicateHandle, std::shared_ptr &inMetadata, const std::string &deviceId); + + int SyncStart(int mode, SingleVerSyncTaskContext *context); + + int TryContinueSync(SingleVerSyncTaskContext *context, const Message *message); + + void ClearSyncStatus(); int PushStart(SingleVerSyncTaskContext *context); @@ -65,6 +81,8 @@ public: int DataRequestRecv(SingleVerSyncTaskContext *context, const Message *message, WaterMark &pullEndWatermark); + bool AckPacketIdCheck(const Message *message); + int AckRecv(SingleVerSyncTaskContext *context, const Message *message); void SendSaveDataNotifyPacket(SingleVerSyncTaskContext *context, uint32_t pktVersion, uint32_t sessionId, @@ -73,8 +91,6 @@ public: virtual int SendDataAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, WaterMark maxSendDataTime); - int32_t ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo); - int CheckPermitSendData(int inMode, SingleVerSyncTaskContext *context); std::string GetLabel() const; @@ -94,6 +110,18 @@ public: void ControlAckErrorHandle(const SingleVerSyncTaskContext *context, const std::shared_ptr &subManager) const; + void PutDataMsg(Message *message); + + Message *MoveNextDataMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, bool &isNeedContinue); + + bool IsNeedReloadQueue(); + + void SendFinishedDataAck(SingleVerSyncTaskContext *context, const Message *message); + + void ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, const Message *message); + + void ClearDataMsg(); + protected: static const int SEND_FINISHED = 0xff; static const int LOCAL_WATER_MARK_NOT_INIT = 0xaa; @@ -101,10 +129,24 @@ protected: static const int WATER_MARK_INVALID = 0xbb; static const int MTU_SIZE = 28311552; // 27MB + void ResetSyncStatus(int inMode, SingleVerSyncTaskContext *context); + + int InnerSyncStart(SingleVerSyncTaskContext *context); + + void InnerClearSyncStatus(); + + int ReSendData(SingleVerSyncTaskContext *context); + + int32_t ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo); + TimeStamp GetMaxSendDataTime(const std::vector &inData); TimeStamp GetMinSendDataTime(const std::vector &inData, WaterMark localMark); + void SetSessionEndTimeStamp(TimeStamp end); + + TimeStamp GetSessionEndTimeStamp() const; + void FillDataRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, SyncEntry &syncData, int sendCode, int mode); @@ -220,6 +262,8 @@ protected: int GetReSendMode(int mode, uint32_t sequenceId, SyncType syncType); + void UpdateSendInfo(SyncTimeRange dataTimeRange, SingleVerSyncTaskContext *context); + void FillRequestReSendPacket(const SingleVerSyncTaskContext *context, DataRequestPacket *packet, DataSyncReSendInfo reSendInfo, SyncEntry &syncData, int sendCode); @@ -260,8 +304,25 @@ protected: std::shared_ptr metadata_; std::string label_; std::string deviceId_; + + SingleVerDataMessageSchedule msgSchedule_; + + static const int HIGH_VERSION_WINDOW_SIZE = 3; + static const int LOW_VERSION_WINDOW_SIZE = 1; + // below param is about sliding sync info, is different from every sync task + std::mutex lock_; + int mode_ = 0; // sync mode, may diff from context mode if trigger pull_response while push finish + uint32_t sessionId_ = 0; + // sequenceId as key + std::map reSendMap_; + // remaining sending window + int32_t windowSize_ = 0; + // max sequenceId has been sent + uint32_t maxSequenceIdHasSent_ = 0; + bool isAllDataHasSent_ = false; + // in a sync session, the last data timeStamp + TimeStamp sessionEndTimeStamp_ = 0; }; } // namespace DistributedDB #endif // SINGLE_VER_DATA_SYNC_NEW_H - diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp deleted file mode 100755 index 4b9926ac6..000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "single_ver_data_sync_with_sliding_window.h" - -namespace DistributedDB { -int SingleVerDataSyncWithSlidingWindow::SenderStart(int32_t mode, SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - return sender_.SendStart(mode, context, dataSync); -} - -int SingleVerDataSyncWithSlidingWindow::SenderAckRecv(const Message *message) -{ - return sender_.AckRecv(message); -} - -int SingleVerDataSyncWithSlidingWindow::PreHandleSenderAckRecv(const Message *message) -{ - return sender_.PreHandleAckRecv(message); -} - -void SingleVerDataSyncWithSlidingWindow::SenderClear() -{ - sender_.Clear(); -} - -void SingleVerDataSyncWithSlidingWindow::SetSenderErr(bool isErr) -{ - sender_.SetErr(isErr); -} - -int SingleVerDataSyncWithSlidingWindow::ReceiverInit(SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - return receiver_.Initialize(context, dataSync); -} - -int SingleVerDataSyncWithSlidingWindow::SenderInit(SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - return sender_.Initialize(context, dataSync); -} - -void SingleVerDataSyncWithSlidingWindow::ReceiverClear() -{ - receiver_.Clear(); -} - -int SingleVerDataSyncWithSlidingWindow::Receive(Message *inMsg) -{ - return receiver_.Receive(inMsg); -} -} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h deleted file mode 100755 index 6d77f0ccf..000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H -#define SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H - -#include "sliding_window_receiver.h" -#include "sliding_window_sender.h" - -namespace DistributedDB { -class SingleVerDataSyncWithSlidingWindow { -public: - SingleVerDataSyncWithSlidingWindow() = default; - ~SingleVerDataSyncWithSlidingWindow() = default; - DISABLE_COPY_ASSIGN_MOVE(SingleVerDataSyncWithSlidingWindow); - - int SenderStart(int32_t mode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int PreHandleSenderAckRecv(const Message *message); - int SenderAckRecv(const Message *message); - void SenderClear(); - void SetSenderErr(bool isErr); - - int ReceiverInit(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int SenderInit(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - void ReceiverClear(); - int Receive(Message *inMsg); -private: - SlidingWindowSender sender_; - SlidingWindowReceiver receiver_; -}; -} // namespace DistributedDB -#endif // SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H - diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp index 0dc5fe000..0208d170e 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp @@ -37,8 +37,9 @@ namespace { const int EVENT_INDEX = 1; const int OUTPUT_STATE_INDEX = 2; - // State switch table v1 and v2, has three columns, CurrentState, Event, and OutSate - const std::vector> STATE_SWITCH_TABLE_V2 = { + // drop v1 and v2 table by one optimize, dataSend mode in all version go with slide window mode. + // State switch table v3, has three columns, CurrentState, Event, and OutSate + const std::vector> STATE_SWITCH_TABLE_V3 = { {IDLE, START_SYNC_EVENT, TIME_SYNC}, // In TIME_SYNC state @@ -48,45 +49,25 @@ namespace { // In ABILITY_SYNC state, compare version num and schema {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, - {ABILITY_SYNC, SWITCH_TO_PROCTOL_V1_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, {ABILITY_SYNC, CONTROL_CMD_EVENT, SYNC_CONTROL_CMD}, // In START_INITIACTIVE_DATA_SYNC state, send a sync request, and send first packt of data sync - {START_INITIACTIVE_DATA_SYNC, SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, - {START_INITIACTIVE_DATA_SYNC, RESPONSE_PUSH_REMAINDER_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_INITIACTIVE_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, {START_INITIACTIVE_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {START_INITIACTIVE_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_INITIACTIVE_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_DATA_SYNC}, {START_INITIACTIVE_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, - - // In INACTIVE_PUSH_REMAINDER_DATA state, do initiactive sync, send remainder pcket of data sync - {INACTIVE_PUSH_REMAINDER_DATA, SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, - {INACTIVE_PUSH_REMAINDER_DATA, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, - {INACTIVE_PUSH_REMAINDER_DATA, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {INACTIVE_PUSH_REMAINDER_DATA, INNER_ERR_EVENT, INNER_ERR}, - {INACTIVE_PUSH_REMAINDER_DATA, RE_SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, - {INACTIVE_PUSH_REMAINDER_DATA, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + {START_INITIACTIVE_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_DATA_SYNC}, // In START_PASSIVE_DATA_SYNC state, do response pull request, and send first packt of data sync - {START_PASSIVE_DATA_SYNC, SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_PASSIVE_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, {START_PASSIVE_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, {START_PASSIVE_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {START_PASSIVE_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_PASSIVE_DATA_SYNC, RE_SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_PASSIVE_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - - // In PASSIVE_PUSH_REMAINDER_DATA state, do passive sync, send remainder pcket of data sync - {PASSIVE_PUSH_REMAINDER_DATA, SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, - {PASSIVE_PUSH_REMAINDER_DATA, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, - {PASSIVE_PUSH_REMAINDER_DATA, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {PASSIVE_PUSH_REMAINDER_DATA, INNER_ERR_EVENT, INNER_ERR}, - {PASSIVE_PUSH_REMAINDER_DATA, RE_SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, - {PASSIVE_PUSH_REMAINDER_DATA, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + {START_PASSIVE_DATA_SYNC, RE_SEND_DATA_EVENT, START_PASSIVE_DATA_SYNC}, // In WAIT_FOR_RECEIVE_DATA_FINISH, {WAIT_FOR_RECEIVE_DATA_FINISH, RECV_FINISHED_EVENT, SYNC_TASK_FINISHED}, @@ -108,58 +89,6 @@ namespace { {SYNC_TIME_OUT, ANY_EVENT, SYNC_TASK_FINISHED}, {INNER_ERR, ANY_EVENT, SYNC_TASK_FINISHED}, }; - - // State switch table v3, has three columns, CurrentState, Event, and OutSate - const std::vector> STATE_SWITCH_TABLE_V3 = { - {IDLE, START_SYNC_EVENT, TIME_SYNC}, - - // In TIME_SYNC state - {TIME_SYNC, TIME_SYNC_FINISHED_EVENT, ABILITY_SYNC}, - {TIME_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {TIME_SYNC, INNER_ERR_EVENT, INNER_ERR}, - - // In ABILITY_SYNC state, compare version num and schema - {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, - {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, - {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {ABILITY_SYNC, CONTROL_CMD_EVENT, SYNC_CONTROL_CMD}, - - // In START_INITIACTIVE_SLIDING_DATA_SYNC state, send a sync request, and send first packt of data sync - {START_INITIACTIVE_SLIDING_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, - - // In START_PASSIVE_SLIDING_DATA_SYNC state, do response pull request, and send first packt of data sync - {START_PASSIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - {START_PASSIVE_SLIDING_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, - {START_PASSIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {START_PASSIVE_SLIDING_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_PASSIVE_SLIDING_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - {START_PASSIVE_SLIDING_DATA_SYNC, RE_SEND_DATA_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - - // In WAIT_FOR_RECEIVE_DATA_FINISH, - {WAIT_FOR_RECEIVE_DATA_FINISH, RECV_FINISHED_EVENT, SYNC_TASK_FINISHED}, - {WAIT_FOR_RECEIVE_DATA_FINISH, START_PULL_RESPONSE_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - {WAIT_FOR_RECEIVE_DATA_FINISH, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {WAIT_FOR_RECEIVE_DATA_FINISH, INNER_ERR_EVENT, INNER_ERR}, - {WAIT_FOR_RECEIVE_DATA_FINISH, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - - {SYNC_CONTROL_CMD, SEND_FINISHED_EVENT, SYNC_TASK_FINISHED}, - {SYNC_CONTROL_CMD, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {SYNC_CONTROL_CMD, INNER_ERR_EVENT, INNER_ERR}, - {SYNC_CONTROL_CMD, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - - // In SYNC_TASK_FINISHED, - {SYNC_TASK_FINISHED, ALL_TASK_FINISHED_EVENT, IDLE}, - {SYNC_TASK_FINISHED, START_SYNC_EVENT, TIME_SYNC}, - - // SYNC_TIME_OUT and INNE_ERR state, just do some exception resolve - {SYNC_TIME_OUT, ANY_EVENT, SYNC_TASK_FINISHED}, - {INNER_ERR, ANY_EVENT, SYNC_TASK_FINISHED}, - }; } std::mutex SingleVerSyncStateMachine::stateSwitchTableLock_; @@ -197,7 +126,6 @@ int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, ISyncInterf timeSync_ = std::make_unique(); dataSync_ = std::make_shared(); abilitySync_ = std::make_unique(); - dataSyncWithSlidingWindow_ = std::make_unique(); errCode = timeSync_->Initialize(communicator, metaData, syncInterface, context->GetDeviceId()); if (errCode != E_OK) { @@ -215,14 +143,7 @@ int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, ISyncInterf currentState_ = IDLE; context_ = static_cast(context); syncInterface_ = static_cast(syncInterface); - errCode = dataSyncWithSlidingWindow_->ReceiverInit(context_, dataSync_); - if (errCode != E_OK) { - goto ERROR_OUT; - } - errCode = dataSyncWithSlidingWindow_->SenderInit(context_, dataSync_); - if (errCode != E_OK) { - goto ERROR_OUT; - } + InitStateSwitchTables(); InitStateMapping(); return E_OK; @@ -324,11 +245,10 @@ void SingleVerSyncStateMachine::AbortInner() { LOGE("[StateMachine][AbortInner] error occurred,abort,label=%s,dev=%s", dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - int mode = SyncOperation::TransferSyncMode(context_->GetMode()); - if (mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL || context_->IsKilled()) { - dataSyncWithSlidingWindow_->ReceiverClear(); + if (context_->IsKilled()) { + dataSync_->ClearDataMsg(); } - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); ContinueToken token; context_->GetContinueToken(token); if (token != nullptr) { @@ -384,7 +304,6 @@ void SingleVerSyncStateMachine::InitStateSwitchTables() return; } - InitStateSwitchTable(SINGLE_VER_SYNC_PROCTOL_V2, STATE_SWITCH_TABLE_V2); InitStateSwitchTable(SINGLE_VER_SYNC_PROCTOL_V3, STATE_SWITCH_TABLE_V3); std::sort(stateSwitchTables_.begin(), stateSwitchTables_.end(), [](const auto &tableA, const auto &tableB) { @@ -419,54 +338,17 @@ void SingleVerSyncStateMachine::InitStateMapping() { stateMapping_[TIME_SYNC] = std::bind(&SingleVerSyncStateMachine::DoTimeSync, this); stateMapping_[ABILITY_SYNC] = std::bind(&SingleVerSyncStateMachine::DoAbilitySync, this); - stateMapping_[START_INITIACTIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveDataSync, - this); - stateMapping_[START_PASSIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoPassiveDataSync, this); - stateMapping_[INACTIVE_PUSH_REMAINDER_DATA] = std::bind(&SingleVerSyncStateMachine::DoInitiactivePushRemainderData, - this); - stateMapping_[PASSIVE_PUSH_REMAINDER_DATA] = std::bind(&SingleVerSyncStateMachine::DoPassivePushRemainderData, - this); stateMapping_[WAIT_FOR_RECEIVE_DATA_FINISH] = std::bind(&SingleVerSyncStateMachine::DoWaitForDataRecv, this); stateMapping_[SYNC_TASK_FINISHED] = std::bind(&SingleVerSyncStateMachine::DoSyncTaskFinished, this); stateMapping_[SYNC_TIME_OUT] = std::bind(&SingleVerSyncStateMachine::DoTimeout, this); stateMapping_[INNER_ERR] = std::bind(&SingleVerSyncStateMachine::DoInnerErr, this); - stateMapping_[START_INITIACTIVE_SLIDING_DATA_SYNC] = + stateMapping_[START_INITIACTIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow, this); - stateMapping_[START_PASSIVE_SLIDING_DATA_SYNC] = + stateMapping_[START_PASSIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow, this); stateMapping_[SYNC_CONTROL_CMD] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveControlSync, this); } -Event SingleVerSyncStateMachine::DoInitiactiveDataSync() -{ - int errCode = E_OK; - int mode = SyncOperation::TransferSyncMode(context_->GetMode()); - switch (mode) { - case SyncModeType::PUSH: - context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); - errCode = dataSync_->PushStart(context_); - break; - case SyncModeType::PULL: - context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - errCode = dataSync_->PullRequestStart(context_); - break; - case SyncModeType::PUSH_AND_PULL: - errCode = dataSync_->PushPullStart(context_); - break; - case SyncModeType::RESPONSE_PULL: - // In response pull mode, reminader data should send in - // PASSIVE_PUSH_REMAINDER_DATA - return Event::RESPONSE_PUSH_REMAINDER_EVENT; - default: - errCode = -E_NOT_SUPPORT; - break; - } - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() { LOGD("[StateMachine][activeDataSync] mode=%d,label=%s,dev=%s", context_->GetMode(), @@ -476,19 +358,19 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() case SyncModeType::PUSH: case SyncModeType::QUERY_PUSH: context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; case SyncModeType::PULL: case SyncModeType::QUERY_PULL: context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; case SyncModeType::PUSH_AND_PULL: case SyncModeType::QUERY_PUSH_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; case SyncModeType::RESPONSE_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; default: errCode = -E_NOT_SUPPORT; @@ -509,23 +391,6 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() return (ignoreInnerErr && event == INNER_ERR_EVENT) ? SEND_FINISHED_EVENT : event; } -Event SingleVerSyncStateMachine::DoPassiveDataSync() -{ - { - RefObject::AutoLock lock(context_); - if (context_->GetRspTargetQueueSize() != 0) { - PreStartPullResponse(); - } else { - return RESPONSE_TASK_FINISHED_EVENT; - } - } - int errCode = dataSync_->PullResponseStart(context_); - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - Event SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow() { { @@ -536,7 +401,7 @@ Event SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow() return RESPONSE_TASK_FINISHED_EVENT; } } - int errCode = dataSyncWithSlidingWindow_->SenderStart(SyncModeType::RESPONSE_PULL, context_, dataSync_); + int errCode = dataSync_->SyncStart(SyncModeType::RESPONSE_PULL, context_); if (errCode != E_OK) { LOGW("[SingleVerSyncStateMachine][DoPassiveDataSyncWithSlidingWindow] response pull send failed[%d]", errCode); return RESPONSE_TASK_FINISHED_EVENT; @@ -569,39 +434,6 @@ int SingleVerSyncStateMachine::HandleControlAckRecv(const Message *inMsg) return E_OK; } -Event SingleVerSyncStateMachine::DoInitiactivePushRemainderData() -{ - int errCode; - int mode = SyncOperation::TransferSyncMode(context_->GetMode()); - switch (mode) { - case SyncModeType::PULL: - case SyncModeType::RESPONSE_PULL: - // In pull or response pul mode, don't need to do INACTIVE_PUSH - return Event::SEND_FINISHED_EVENT; - case SyncModeType::PUSH: - case SyncModeType::PUSH_AND_PULL: - errCode = dataSync_->PushStart(context_); - break; - default: - errCode = -E_INTERNAL_ERROR; - break; - } - - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - -Event SingleVerSyncStateMachine::DoPassivePushRemainderData() -{ - int errCode = dataSync_->PullResponseStart(context_); - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - Event SingleVerSyncStateMachine::DoWaitForDataRecv() const { if (context_->GetRspTargetQueueSize() != 0) { @@ -676,17 +508,13 @@ Event SingleVerSyncStateMachine::GetEventAfterTimeSync(int mode, bool isEarliest if (mode == SyncModeType::SUBSCRIBE_QUERY || mode == SyncModeType::UNSUBSCRIBE_QUERY) { return Event::CONTROL_CMD_EVENT; } - if (isEarliestVersion) { - LOGI("remote version is 0, switch to v1 proctol"); - return Event::SWITCH_TO_PROCTOL_V1_EVENT; - } return Event::ABILITY_SYNC_FINISHED_EVENT; } Event SingleVerSyncStateMachine::DoSyncTaskFinished() { StopWatchDog(); - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); RefObject::AutoLock lock(syncContext_); int errCode = ExecNextTask(); if (errCode == E_OK) { @@ -786,6 +614,16 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) { + TimeOffset offset = 0; + uint32_t timeout = TIME_SYNC_WAIT_TIME; + timeout = communicator_->GetTimeout(context_->GetDeviceId()); + // If message is data sync request, we should check timeoffset. + int errCode = timeSync_->GetTimeOffset(offset, timeout); + if (errCode != E_OK) { + LOGE("[StateMachine][HandleDataRequestRecv] GetTimeOffset err! errCode=%d", errCode); + return errCode; + } + context_->SetTimeOffset(offset); PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); if (performance != nullptr) { performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); @@ -802,7 +640,7 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) // So we need to send save data notify to keep remote alive. bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); WaterMark pullEndWaterkark = 0; - int errCode = dataSync_->DataRequestRecv(context_, inMsg, pullEndWaterkark); + errCode = dataSync_->DataRequestRecv(context_, inMsg, pullEndWaterkark); if (performance != nullptr) { performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); } @@ -823,22 +661,14 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) return E_OK; } -int SingleVerSyncStateMachine::PreHandleAckRecv(const Message *inMsg) -{ - if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - return dataSyncWithSlidingWindow_->PreHandleSenderAckRecv(inMsg); - } - return E_OK; -} - void SingleVerSyncStateMachine::HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg, bool ignoreInnerErr) { if (errCode == -E_RE_SEND_DATA) { // LOCAL_WATER_MARK_NOT_INIT - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); } if (errCode == -E_NO_DATA_SEND || errCode == -E_SEND_DATA) { - int ret = dataSyncWithSlidingWindow_->SenderAckRecv(inMsg); + int ret = dataSync_->TryContinueSync(context_, inMsg); if (ret == -E_FINISHED) { SwitchStateAndStep(Event::SEND_FINISHED_EVENT); return; @@ -862,7 +692,7 @@ void SingleVerSyncStateMachine::NeedAbilitySyncHandle() currentRemoteVersionId_ = context_->GetRemoteSoftwareVersionId(); } abilitySync_->SetAbilitySyncFinishedStatus(false); - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); } int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) @@ -881,22 +711,16 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) if (IsNeedResetWatchdog(inMsg)) { (void)ResetWatchDog(); } - int errCode = PreHandleAckRecv(inMsg); - if (errCode != E_OK) { + if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && !dataSync_->AckPacketIdCheck(inMsg)) { // packetId not match but sequence id matched scene, means resend map has be rebuilt // this is old ack, shoulb be dropped and wait for the same packetId sequence. - if (errCode == -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG) { - return E_OK; - } - // this means error happened,should stop the sync task. - SwitchStateAndStep(TransformErrCodeToEvent(errCode)); - return errCode; + return E_OK; } // AckRecv will save meta data, it may cost a long time. if another thread is saving data // So we need to send save data notify to keep remote alive. // eg. remote do pull sync bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); - errCode = dataSync_->AckRecv(context_, inMsg); + int errCode = dataSync_->AckRecv(context_, inMsg); if (isNeedStop) { StopSaveDataNotify(); } @@ -909,10 +733,6 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) bool ignoreInnerErr = inMsg->GetSessionId() == context_->GetResponseSessionId() && context_->GetRequestSessionId() != 0; DataAckRecvErrCodeHandle(errCode, !ignoreInnerErr); - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - ResponsePullError(errCode, ignoreInnerErr); - return errCode; - } HandleDataAckRecvWithSlidingWindow(errCode, inMsg, ignoreInnerErr); return errCode; } @@ -920,26 +740,11 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) int SingleVerSyncStateMachine::DataPktRecv(Message *inMsg) { PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - int errCode; - TimeOffset offset = 0; - uint32_t timeout = TIME_SYNC_WAIT_TIME; - + int errCode = E_OK; switch (inMsg->GetMessageType()) { case TYPE_REQUEST: - timeout = communicator_->GetTimeout(context_->GetDeviceId()); - // If message is data sync request, we should check timeoffset. - errCode = timeSync_->GetTimeOffset(offset, timeout); - if (errCode != E_OK) { - LOGE("[StateMachine][DataPktRecv] GetTimeOffset err! errCode=%d", errCode); - return errCode; - } - context_->SetTimeOffset(offset); - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - // higher than 102 version may go to here, but it is ok not use slwr,after abilitysync would go slwr. - errCode = HandleDataRequestRecv(inMsg); - } else { - errCode = dataSyncWithSlidingWindow_->Receive(inMsg); - } + ScheduleMsgAndHandle(inMsg); + errCode = -E_NOT_NEED_DELETE_MSG; break; case TYPE_RESPONSE: case TYPE_NOTIFY: @@ -956,6 +761,43 @@ int SingleVerSyncStateMachine::DataPktRecv(Message *inMsg) return errCode; } +void SingleVerSyncStateMachine::ScheduleMsgAndHandle(Message *inMsg) +{ + dataSync_->PutDataMsg(inMsg); + while (true) { + bool isNeedHandle = true; + bool isNeedContinue = true; + Message *msg = dataSync_->MoveNextDataMsg(context_, isNeedHandle, isNeedContinue); + if (!isNeedContinue) { + break; + } + if (msg == nullptr) { + if (dataSync_->IsNeedReloadQueue()) { + continue; + } + break; + } + bool isNeedClearMap = false; + if (isNeedHandle) { + int errCode = HandleDataRequestRecv(msg); + if (context_->IsReceiveWaterMarkErr() || errCode == -E_NEED_ABILITY_SYNC) { + isNeedClearMap = true; + } + if (errCode == -E_TIMEOUT) { + isNeedHandle = false; + } + } else { + dataSync_->SendFinishedDataAck(context_, msg); + } + if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { + // for lower version, no need to handle map schedule info, just reset schedule working status + isNeedHandle = false; + } + dataSync_->ScheduleInfoHandle(isNeedHandle, isNeedClearMap, msg); + delete msg; + } +} + int SingleVerSyncStateMachine::ControlPktRecv(Message *inMsg) { int errCode = E_OK; @@ -1034,7 +876,6 @@ int SingleVerSyncStateMachine::TimeMarkSyncRecv(const Message *inMsg) void SingleVerSyncStateMachine::Clear() { - dataSyncWithSlidingWindow_ = nullptr; dataSync_ = nullptr; timeSync_ = nullptr; abilitySync_ = nullptr; @@ -1073,18 +914,7 @@ bool SingleVerSyncStateMachine::IsPacketValid(const Message *inMsg) const inMsg->GetMessageId() == CONTROL_SYNC_MESSAGE) { return true; } - - // sliding window don't need to check sequenceId here, just return - if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - return true; - } - - if (isResponseType && (inMsg->GetSequenceId() != context_->GetSequenceId())) { - LOGE("[StateMachine][IsPacketValid] Message is invalid,inMsg SequenceId=%d,seqId=%d,syncId=%d", - inMsg->GetSequenceId(), context_->GetSequenceId(), context_->GetSyncId()); - return false; - } - + // sequenceId will be checked in dataSync return true; } @@ -1100,23 +930,6 @@ void SingleVerSyncStateMachine::PreStartPullResponse() context_->SetQuery(target.GetQuery()); } -bool SingleVerSyncStateMachine::IsRightDataResponsePkt(const Message *inMsg) const -{ - if (inMsg->GetMessageId() != DATA_SYNC_MESSAGE || inMsg->GetMessageId() != QUERY_SYNC_MESSAGE) { - return false; - } - - if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - return false; - } - - if ((context_->GetMode() != SyncModeType::INVALID_MODE) && (inMsg->GetMessageType() == TYPE_RESPONSE)) { - return true; - } - - return false; -} - bool SingleVerSyncStateMachine::CheckIsStartPullResponse() const { // Other state will step to do pull response, only this statem we need to step the statemachine @@ -1136,10 +949,6 @@ int SingleVerSyncStateMachine::MessageCallbackPre(const Message *inMsg) if (!IsPacketValid(inMsg)) { return -E_INVALID_ARGS; } - - if (IsRightDataResponsePkt(inMsg)) { - context_->IncSequenceId(); - } return E_OK; } @@ -1238,11 +1047,6 @@ Event SingleVerSyncStateMachine::TransforTimeOutErrCodeToEvent() } } -void SingleVerSyncStateMachine::SetSlidingWindowSenderErr(bool isErr) -{ - dataSyncWithSlidingWindow_->SetSenderErr(isErr); -} - bool SingleVerSyncStateMachine::IsNeedErrCodeHandle(uint32_t sessionId) const { // omit to set sessionId so version_3 should skip to compare sessionid. @@ -1320,12 +1124,6 @@ void SingleVerSyncStateMachine::DataAckRecvErrCodeHandle(int errCode, bool handl case -E_NEED_ABILITY_SYNC: NeedAbilitySyncHandle(); break; - case -E_NO_DATA_SEND: - // when version is higher than 102, this step will be handle in slide windows function. - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0 && handleError) { - context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - } - break; case -E_NOT_PERMIT: if (handleError) { context_->SetOperationStatus(SyncOperation::OP_PERMISSION_CHECK_FAILED); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h index 5fbb258f1..965c13eeb 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h @@ -24,7 +24,6 @@ #include "meta_data.h" #include "semaphore_utils.h" #include "single_ver_data_sync.h" -#include "single_ver_data_sync_with_sliding_window.h" #include "single_ver_sync_task_context.h" #include "sync_state_machine.h" #include "sync_target.h" @@ -38,16 +37,12 @@ namespace { IDLE = 0, TIME_SYNC, ABILITY_SYNC, - START_INITIACTIVE_DATA_SYNC, // used to do sync started by local device - START_PASSIVE_DATA_SYNC, // used to do sync response remote device - INACTIVE_PUSH_REMAINDER_DATA, // push remainded data if initactive sync data more than on frame - PASSIVE_PUSH_REMAINDER_DATA, // push remainded data if passive sync data more than on frame WAIT_FOR_RECEIVE_DATA_FINISH, // all data send finished, wait for data revice if has pull request SYNC_TASK_FINISHED, // current sync task finihsed, try to schedule next sync task SYNC_TIME_OUT, INNER_ERR, - START_INITIACTIVE_SLIDING_DATA_SYNC, // used to do sync started by local device, use sliding window - START_PASSIVE_SLIDING_DATA_SYNC, // used to do pull response, use sliding window + START_INITIACTIVE_DATA_SYNC, // used to do sync started by local device, use sliding window + START_PASSIVE_DATA_SYNC, // used to do pull response, use sliding window SYNC_CONTROL_CMD // used to send control cmd. }; @@ -56,12 +51,10 @@ namespace { TIME_SYNC_FINISHED_EVENT, ABILITY_SYNC_FINISHED_EVENT, VERSION_NOT_SUPPOR_EVENT, - SWITCH_TO_PROCTOL_V1_EVENT, SEND_DATA_EVENT, SEND_FINISHED_EVENT, RECV_FINISHED_EVENT, NEED_ABILITY_SYNC_EVENT, - RESPONSE_PUSH_REMAINDER_EVENT, RESPONSE_TASK_FINISHED_EVENT, START_PULL_RESPONSE_EVENT, WAIT_ACK_EVENT, @@ -94,8 +87,6 @@ public: int HandleDataRequestRecv(const Message *inMsg); - void SetSlidingWindowSenderErr(bool isErr); - bool IsNeedErrCodeHandle(uint32_t sessionId) const; void PushPullDataRequestEvokeErrHandle(); @@ -151,18 +142,6 @@ private: // Do AbilitySync, for first sync Event DoAbilitySync(); - // Do data packet sync for initiactive sync operation, if need pull, there need send a pull request. - Event DoInitiactiveDataSync(); - - // Do data packet sync for response pull request. - Event DoPassiveDataSync(); - - // Push remainder data to remote - Event DoInitiactivePushRemainderData(); - - // Push remainder data to remote for response pull request - Event DoPassivePushRemainderData(); - // Waiting for pull data revice finish, if coming a pull request, should goto START_PASSIVE_DATA_SYNC state Event DoWaitForDataRecv() const; @@ -193,14 +172,14 @@ private: int DataPktRecv(Message *inMsg); + void ScheduleMsgAndHandle(Message *inMsg); + int ControlPktRecv(Message *inMsg); void NeedAbilitySyncHandle(); int HandleDataAckRecv(const Message *inMsg); - int PreHandleAckRecv(const Message *inMsg); - void HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg, bool ignoreInnerErr); void ResponsePullError(int errCode, bool ignoreInnerErr); @@ -211,8 +190,6 @@ private: void PreStartPullResponse(); - bool IsRightDataResponsePkt(const Message *inMsg) const; - bool CheckIsStartPullResponse() const; int MessageCallbackPre(const Message *inMsg); @@ -245,7 +222,6 @@ private: std::unique_ptr timeSync_; std::unique_ptr abilitySync_; std::shared_ptr dataSync_; - std::unique_ptr dataSyncWithSlidingWindow_; uint64_t currentRemoteVersionId_; std::map stateMapping_; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp index d07b32898..cbc1fbb56 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp @@ -307,27 +307,6 @@ void SingleVerSyncTaskContext::StopFeedDogForSync(SyncDirectionFlag flag) stateMachine_->StopFeedDogForSync(flag); } -void SingleVerSyncTaskContext::SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange) -{ - dataTimeRange_ = dataTimeRange; -} - -SyncTimeRange SingleVerSyncTaskContext::GetDataTimeRange() const -{ - return dataTimeRange_; -} - - -void SingleVerSyncTaskContext::SetSessionEndTimeStamp(TimeStamp end) -{ - sessionEndTimeStamp_ = end; -} - -TimeStamp SingleVerSyncTaskContext::GetSessionEndTimeStamp() const -{ - return sessionEndTimeStamp_; -} - int SingleVerSyncTaskContext::HandleDataRequestRecv(const Message *msg) { return static_cast(stateMachine_)->HandleDataRequestRecv(msg); @@ -343,11 +322,6 @@ void SingleVerSyncTaskContext::SetReceiveWaterMarkErr(bool isErr) isReceiveWaterMarkErr_ = isErr; } -void SingleVerSyncTaskContext::SetSlidingWindowSenderErr(bool isErr) -{ - static_cast(stateMachine_)->SetSlidingWindowSenderErr(isErr); -} - void SingleVerSyncTaskContext::SetRemoteSeccurityOption(SecurityOption secOption) { remoteSecOption_ = secOption; @@ -546,7 +520,7 @@ bool SingleVerSyncTaskContext::IsCurrentSyncTaskCanBeSkipped() const return false; } if (localWaterMark > maxTimeStampInDb) { - LOGD("skip current push task, deviceId_ = %s, localWaterMark = %llu, maxTimeStampInDb = %llu", + LOGI("skip current push task, deviceId_ = %s, localWaterMark = %llu, maxTimeStampInDb = %llu", STR_MASK(deviceId_), localWaterMark, maxTimeStampInDb); return true; } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h index 221d65e99..85f29c212 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h @@ -88,28 +88,14 @@ public: // stop timer to ResetWatchDog when sync data one (key,value) size bigger than mtu void StopFeedDogForSync(SyncDirectionFlag flag); - SyncTimeRange GetDataTimeRange() const; - virtual int HandleDataRequestRecv(const Message *msg); - // if sended by sliding window, set the start and and timeStamp of data in a sequence - void SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange); - - // if sended by sliding window, set the last data timeStamp in a sync session - void SetSessionEndTimeStamp(TimeStamp end); - - // if sended by sliding window, get the last data timeStamp in a sync session - TimeStamp GetSessionEndTimeStamp() const; - // is receive warterMark err bool IsReceiveWaterMarkErr() const; // set receive warterMark err void SetReceiveWaterMarkErr(bool isErr); - // set sliding window sender err - void SetSlidingWindowSenderErr(bool isErr); - void SetRemoteSeccurityOption(SecurityOption secOption); SecurityOption GetRemoteSeccurityOption() const; @@ -173,11 +159,6 @@ private: SyncStrategy syncStrategy_; bool isSchemaSync_ = false; - // normal data or delete data start timestamp and end timestamp,recorded for slws resend. - SyncTimeRange dataTimeRange_; - // in a sync session, the last data timeStamp - TimeStamp sessionEndTimeStamp_ = 0; - // is receive waterMark err, peerWaterMark bigger than remote localWaterMark bool isReceiveWaterMarkErr_ = false; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp deleted file mode 100755 index 3cb5c8a72..000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sliding_window_receiver.h" -#include "sync_task_context.h" -#include "db_common.h" - -namespace DistributedDB { -SlidingWindowReceiver::~SlidingWindowReceiver() -{ - Clear(); - dataSync_ = nullptr; - context_ = nullptr; -} - -void SlidingWindowReceiver::Clear() -{ - StopTimer(); - std::lock_guard lock(lock_); - ClearMap(); - isWaterMarkErrHappened_ = false; -} - -int SlidingWindowReceiver::Initialize(SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - if (context == nullptr || dataSync == nullptr) { - LOGE("[slwr] Initialize invalid args"); - return -E_INVALID_ARGS; - } - context_ = context; - dataSync_ = dataSync; - return E_OK; -} - -int SlidingWindowReceiver::Receive(Message *inMsg) -{ - LOGD("[slwr] receive msg"); - if (inMsg == nullptr) { - return -E_INVALID_ARGS; - } - int errCode = PutMsg(inMsg); - if (errCode == E_OK) { - StopTimer(); - StartTimer(); - DealMsg(); - return -E_NOT_NEED_DELETE_MSG; - } - return errCode; -} - -int SlidingWindowReceiver::PutMsg(Message *inMsg) -{ - const DataRequestPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - uint32_t sessionId = inMsg->GetSessionId(); - uint32_t sequenceId = inMsg->GetSequenceId(); - bool isLastSequence = packet->IsLastSequence(); - std::unique_lock lock(lock_); - if (workingId_ != 0 && sessionId_ != 0) { - LOGI("[PutMsg] task is running, wait for workdingId=%u end,seId=%u", workingId_, sequenceId); - workingTaskcv_.wait(lock, [this] { return workingId_ == 0; }); - } - if (sessionId_ != sessionId) { - ResetInfo(); - sessionId_ = sessionId; - messageMap_[sequenceId] = inMsg; - SetEndField(isLastSequence, sequenceId); - return E_OK; - } - // maybe remote has not receive ack, we resend ack. - if (sequenceId <= hasFinishedMaxId_) { - LOGI("[slwr] seId=%u,FinishedMId_=%u,label=%s", sequenceId, hasFinishedMaxId_, dataSync_->GetLabel().c_str()); - lock.unlock(); - dataSync_->SendDataAck(context_, inMsg, E_OK, 0); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - int errCode = ErrHandle(sequenceId, packet->GetPacketId()); - if (errCode != E_OK) { - return errCode; - } - if (messageMap_.count(sequenceId) > 0) { - LOGI("[slwr] PutMsg sequenceId already in map label=%s,deviceId=%s", - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - // it cannot be null here - const auto *cachePacket = messageMap_[sequenceId]->GetObject(); - if (cachePacket->GetPacketId() > packet->GetPacketId()) { - LOGI("[slwr] PutMsg receive old packet %u, new packet is %u", - packet->GetPacketId(), cachePacket->GetPacketId()); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - delete messageMap_[sequenceId]; - messageMap_[sequenceId] = nullptr; - } - messageMap_[sequenceId] = inMsg; - SetEndField(isLastSequence, sequenceId); - return E_OK; -} - -void SlidingWindowReceiver::DealMsg() -{ - while (true) { - Message *msg = nullptr; - uint64_t curPacketId = 0; - { - std::lock_guard lock(lock_); - if (workingId_ != 0 || messageMap_.count(hasFinishedMaxId_ + 1) == 0) { - LOGI("[slwr] DealMsg do nothing workingId_=%u,hasFinishedMaxId_=%u,label=%s,deviceId=%s", - workingId_, hasFinishedMaxId_, dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - return; - } - uint32_t curWorkingId = hasFinishedMaxId_ + 1; - msg = messageMap_[curWorkingId]; - messageMap_.erase(curWorkingId); - const auto *packet = msg->GetObject(); - curPacketId = packet->GetPacketId(); - if (curPacketId != 0 && curPacketId < hasFinishedPacketId_) { - delete msg; - msg = nullptr; - LOGI("[slwr] DealMsg ignore msg, packetId:%u,label=%s,deviceId=%s", - curPacketId, dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - continue; - } - workingId_ = curWorkingId; - LOGI("[slwr] DealMsg workingId_=%u,label=%s,deviceId=%s", workingId_, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - } - int errCode = context_->HandleDataRequestRecv(msg); - delete msg; - msg = nullptr; - { - std::lock_guard lock(lock_); - workingId_ = 0; - bool isWaterMarkErr = context_->IsReceiveWaterMarkErr(); - if (isWaterMarkErr || errCode == -E_NEED_ABILITY_SYNC) { - ClearMap(); - hasFinishedMaxId_ = 0; - endId_ = 0; - isWaterMarkErrHappened_ = true; - } else { - hasFinishedMaxId_++; - hasFinishedPacketId_ = curPacketId; - LOGI("[slwr] DealMsg ok hasFinishedMaxId_=%u,label=%s,deviceId=%s", hasFinishedMaxId_, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - } - context_->SetReceiveWaterMarkErr(false); - workingTaskcv_.notify_all(); - } - } -} - -int SlidingWindowReceiver::TimeOut(TimerId timerId) -{ - { - std::lock_guard lock(lock_); - LOGI("[slwr] TimeOut,timerId[%llu], timerId_[%llu]", timerId, timerId_); - if (timerId == timerId_) { - ClearMap(); - } - timerId_ = 0; - } - RuntimeContext::GetInstance()->RemoveTimer(timerId); - return E_OK; -} - -void SlidingWindowReceiver::StartTimer() -{ - std::lock_guard lock(lock_); - TimerId timerId = 0; - RefObject::IncObjRef(context_); - TimerAction timeOutCallback = std::bind(&SlidingWindowReceiver::TimeOut, this, std::placeholders::_1); - int errCode = RuntimeContext::GetInstance()->SetTimer(IDLE_TIME_OUT, timeOutCallback, - [this]() { - int ret = RuntimeContext::GetInstance()->ScheduleTask([this]() { - RefObject::DecObjRef(context_); - }); - if (ret != E_OK) { - LOGE("[slwr] timer finalizer ScheduleTask, errCode=%d", ret); - } - }, timerId); - if (errCode != E_OK) { - RefObject::DecObjRef(context_); - LOGE("[slwr] timer ScheduleTask, errCode=%d", errCode); - return; - } - StopTimer(timerId_); - timerId_ = timerId; - LOGD("[slwr] StartTimer timerId[%llu] success", timerId_); -} - -void SlidingWindowReceiver::StopTimer() -{ - TimerId timerId; - { - std::lock_guard lock(lock_); - timerId = timerId_; - if (timerId_ == 0) { - return; - } - timerId_ = 0; - } - StopTimer(timerId); -} - -void SlidingWindowReceiver::StopTimer(TimerId timerId) -{ - if (timerId == 0) return; - LOGD("[slwr] StopTimer,remove Timer id[%llu]", timerId); - RuntimeContext::GetInstance()->RemoveTimer(timerId); -} - -void SlidingWindowReceiver::ClearMap() -{ - LOGD("[slwr] ClearMap"); - for (auto &iter : messageMap_) { - delete iter.second; - iter.second = nullptr; - } - messageMap_.clear(); -} - -void SlidingWindowReceiver::ResetInfo() -{ - ClearMap(); - hasFinishedMaxId_ = 0; - hasFinishedPacketId_ = 0; - workingId_ = 0; - endId_ = 0; - isWaterMarkErrHappened_ = false; -} - -int SlidingWindowReceiver::ErrHandle(uint32_t sequenceId, uint64_t packetId) -{ - if (sequenceId == workingId_ || (endId_ != 0 && sequenceId > endId_) - || (packetId > 0 && packetId < hasFinishedPacketId_)) { - LOGI("[slwr] PutMsg sequenceId:%u, endId_:%u, workingId_:%u, packetId:%u!", - sequenceId, endId_, workingId_, packetId); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - // if waterMark err when DealMsg(), the waterMark of msg in messageMap_ is also err, so we clear messageMap_ - // but before the sender receive the waterMark err ack, the receiver may receive waterMark err msg, so before - // receiver receive the sequenceId 1 msg, it will drop msg. - // note, there is a low probability risk, if sender receive the waterMark err ack, it correct the warterMark then - // resend data, if the msg of bigger sequenceId arrive earlier than sequenceId 1, it will be drop, the sender - // will sync timeout. - if (isWaterMarkErrHappened_) { - if (sequenceId == 1) { - isWaterMarkErrHappened_ = false; - } else { - LOGI("[slwr] PutMsg With waterMark Error sequenceId:%u, endId_:%u, workingId_:%u, packetId:%u!", - sequenceId, endId_, workingId_, packetId); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; // drop invalid packet - } - } - return E_OK; -} - -void SlidingWindowReceiver::SetEndField(bool isLastSequence, uint32_t sequenceId) -{ - if (isLastSequence) { - endId_ = sequenceId; - } -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp deleted file mode 100755 index 32df1ca76..000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sliding_window_sender.h" -#include "db_common.h" -namespace DistributedDB { -SlidingWindowSender::~SlidingWindowSender() -{ - Clear(); - context_ = nullptr; - dataSync_ = nullptr; -} - -int SlidingWindowSender::ParamCheck(int32_t mode, const SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - if (context == nullptr) { - LOGE("[slws] SendStart invalid context"); - return -E_INVALID_ARGS; - } - if (dataSync == nullptr) { - LOGE("[slws] SendStart invalid context"); - return -E_INVALID_ARGS; - } - return E_OK; -} - -void SlidingWindowSender::Init(int32_t inMode, SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - isErr_ = false; - mode_ = inMode; - context_->SetSessionEndTimeStamp(0); - context_->ReSetSequenceId(); - maxSequenceIdhasSent_ = 0; - windowSize_ = MAX_WINDOW_SIZE; - int mode = SyncOperation::TransferSyncMode(inMode); - if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL) { - sessionId_ = context->GetRequestSessionId(); - } else { - sessionId_ = context->GetResponseSessionId(); - } -} - -int SlidingWindowSender::Initialize(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync) -{ - if (context == nullptr || dataSync == nullptr) { - LOGE("[slws] Initialize invalid args"); - return -E_INVALID_ARGS; - } - context_ = context; - dataSync_ = dataSync; - return E_OK; -} - -int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - int errCode = ParamCheck(mode, context, dataSync); - if (errCode != E_OK) { - return errCode; - } - errCode = dataSync->CheckPermitSendData(mode, context); - if (errCode != E_OK) { - return errCode; - } - std::lock_guard lock(lock_); - if (sessionId_ != 0) { // auto sync timeout resend - return ReSend(); - } - Init(mode, context, dataSync); - LOGI("[slws] SendStart,mode=%d,label=%s,device=%s", mode_, dataSync->GetLabel().c_str(), - STR_MASK(context->GetDeviceId())); - int tmpMode = SyncOperation::TransferSyncMode(mode); - if (tmpMode == SyncModeType::PUSH) { - errCode = dataSync->PushStart(context); - } else if (tmpMode == SyncModeType::PUSH_AND_PULL) { - errCode = dataSync->PushPullStart(context); - } else if (tmpMode == SyncModeType::PULL) { - errCode = dataSync->PullRequestStart(context); - } else { - errCode = dataSync->PullResponseStart(context); - } - if (context->IsSkipTimeoutError(errCode)) { - // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. - // just return to avoid higher pressure for send. - UpdateInfo(); - return errCode; - } - if (errCode != E_OK) { - LOGE("[slws] SendStart errCode=%d", errCode); - return errCode; - } - errCode = UpdateInfo(); - if (tmpMode == SyncModeType::PUSH_AND_PULL && context_->GetTaskErrCode() == -E_EKEYREVOKED) { - LOGE("wait for recv finished for push and pull mode"); - return -E_EKEYREVOKED; - } - if (errCode == -E_FINISHED) { - return E_OK; - } - return InnerSend(); -} - -int SlidingWindowSender::UpdateInfo() -{ - ReSendInfo reSendInfo; - SyncTimeRange reSendDataTimeRange = context_->GetDataTimeRange(); - reSendInfo.start = reSendDataTimeRange.beginTime; - reSendInfo.end = reSendDataTimeRange.endTime; - reSendInfo.deleteBeginTime = reSendDataTimeRange.deleteBeginTime; - reSendInfo.deleteEndTime = reSendDataTimeRange.deleteEndTime; - reSendInfo.packetId = context_->GetPacketId(); - maxSequenceIdhasSent_++; - reSendMap_[maxSequenceIdhasSent_] = reSendInfo; - windowSize_--; - LOGI("[slws] mode=%d,start=%llu,end=%llu,deleteStart=%llu,deleteEnd=%llu,seqId=%d,packetId=%llu,window_size=%d," - "label=%s,device=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, - reSendInfo.deleteEndTime, maxSequenceIdhasSent_, reSendInfo.packetId, windowSize_, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - ContinueToken token; - context_->GetContinueToken(token); - if (token == nullptr) { - isAllDataHasSent_ = true; - LOGI("[slws] UpdateInfo all data has sent"); - return -E_FINISHED; - } - return E_OK; -} - -int SlidingWindowSender::InnerSend() -{ - while (true) { - if (windowSize_ <= 0 || isAllDataHasSent_) { - LOGI("[slws] InnerSend windowSize_=%d,isAllDataHasSent_=%d", windowSize_, isAllDataHasSent_); - return E_OK; - } - if (isErr_ || context_ == nullptr) { - LOGI("[slws] InnerSend isErr"); - return -E_SLIDING_WINDOW_SENDER_ERR; - } - int errCode; - context_->IncSequenceId(); - int mode = SyncOperation::TransferSyncMode(mode_); - if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { - errCode = dataSync_->PushStart(context_); - } else { - errCode = dataSync_->PullResponseStart(context_); - } - if ((mode == SyncModeType::PUSH_AND_PULL) && errCode == -E_EKEYREVOKED) { - LOGE("errCode = %d, wait for recv finished for push and pull mode", errCode); - isAllDataHasSent_ = true; - return -E_EKEYREVOKED; - } - if (context_->IsSkipTimeoutError(errCode)) { - // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. - // just return to avoid higher pressure for send. - UpdateInfo(); - return errCode; - } - if (errCode != E_OK) { - isErr_ = true; - LOGE("[slws] InnerSend errCode=%d", errCode); - return errCode; - } - errCode = UpdateInfo(); - if (errCode == -E_FINISHED) { - return E_OK; - } - } -} - -int SlidingWindowSender::PreHandleAckRecv(const Message *message) -{ - if (message == nullptr) { - LOGE("[slws] AckRecv message nullptr"); - return -E_INVALID_ARGS; - } - if (message->GetMessageType() == TYPE_NOTIFY || message->IsFeedbackError()) { - return E_OK; - } - const DataAckPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value - std::lock_guard lock(lock_); - uint32_t sequenceId = message->GetSequenceId(); - if (reSendMap_.count(sequenceId) != 0) { - uint64_t originalPacketId = reSendMap_[sequenceId].packetId; - if (DataAckPacket::IsPacketIdValid(packetId) && packetId != originalPacketId) { - LOGE("[slws] packetId[%llu] is not match with original[%llu]", packetId, originalPacketId); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - } - return E_OK; -} - -int SlidingWindowSender::AckRecv(const Message *message) -{ - if (message == nullptr) { - LOGE("[slws] AckRecv message nullptr"); - return -E_INVALID_ARGS; - } - const DataAckPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value - uint32_t sessionId = message->GetSessionId(); - uint32_t sequenceId = message->GetSequenceId(); - LOGI("[slws] AckRecv sequecneId=%d,packetId=%llu,label=%s,dev=%s", sequenceId, packetId, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - - std::lock_guard lock(lock_); - if (sessionId != sessionId_) { - LOGI("[slws] AckRecv sessionId is different"); - return E_OK; - } - if (reSendMap_.count(sequenceId) != 0) { - reSendMap_.erase(sequenceId); - windowSize_++; - LOGI("[slws] AckRecv window_size=%d", windowSize_); - } else { - LOGI("[slws] AckRecv sequenceId not in map"); - return E_OK; - } - if (!isAllDataHasSent_) { - return InnerSend(); - } else if (reSendMap_.size() == 0) { - context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - LOGI("[slws] AckRecv all finished,label=%s,dev=%s", dataSync_->GetLabel().c_str(), - STR_MASK(context_->GetDeviceId())); - InnerClear(); - return -E_FINISHED; - } - return E_OK; -} - -int SlidingWindowSender::ReSend() const -{ - if (isErr_) { - LOGI("[slws] ReSend InnerSend isErr"); - return -E_INTERNAL_ERROR; - } - if (reSendMap_.empty()) { - LOGI("[slws] ReSend map empty"); - return -E_INTERNAL_ERROR; - } - uint32_t sequenceId = reSendMap_.begin()->first; - ReSendInfo reSendInfo = reSendMap_.begin()->second; - LOGI("[slws] ReSend mode=%d,start=%llu,end=%llu,delStart=%llu,delEnd=%llu,seqId=%d,packetId=%llu,windowsize=%d," - "label=%s,deviceId=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, - reSendInfo.deleteEndTime, sequenceId, reSendInfo.packetId, windowSize_, dataSync_->GetLabel().c_str(), - STR_MASK(context_->GetDeviceId())); - DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, - reSendInfo.deleteBeginTime, reSendInfo.deleteEndTime, reSendInfo.packetId}; - return dataSync_->ReSend(context_, dataReSendInfo); -} - -void SlidingWindowSender::Clear() -{ - std::lock_guard lock(lock_); - LOGD("[slws] clear sender info"); - InnerClear(); -} - -void SlidingWindowSender::InnerClear() -{ - sessionId_ = 0; - reSendMap_.clear(); - windowSize_ = 0; - maxSequenceIdhasSent_ = 0; - isAllDataHasSent_ = false; - isErr_ = false; -} - -void SlidingWindowSender::SetErr(bool isErr) -{ - std::lock_guard lock(lock_); - isErr_ = isErr; -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h deleted file mode 100755 index 652a20a18..000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SLIDING_WINDOW_SENDER_H -#define SLIDING_WINDOW_SENDER_H -#include - -#include "message.h" -#include "single_ver_data_sync.h" -#include "single_ver_sync_task_context.h" - -namespace DistributedDB { -struct ReSendInfo { - TimeStamp start = 0; - TimeStamp end = 0; - TimeStamp deleteBeginTime = 0; - TimeStamp deleteEndTime = 0; - // packetId is used for matched ackpacket packetId which saved in ackPacket.reserve - // if equaled, means need to handle the ack, or drop. it is always increased - uint64_t packetId = 0; -}; - -class SlidingWindowSender { -public: - SlidingWindowSender() = default; - ~SlidingWindowSender(); - DISABLE_COPY_ASSIGN_MOVE(SlidingWindowSender); - - int Initialize(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - // start one dataSync, mode can be push pushAndPull or pullResponse - int SendStart(int32_t mode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int AckRecv(const Message *message); - int PreHandleAckRecv(const Message *message); - void Clear(); - void SetErr(bool isErr); -private: - int ParamCheck(int32_t mode, const SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync); - void Init(int32_t inMode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int UpdateInfo(); - int InnerSend(); - void InnerClear(); - int ReSend() const; // when timeout, used for reSend data - - // initial max window size - static const int MAX_WINDOW_SIZE = 3; - std::mutex lock_; - // 0 is default invalid sessionId. - uint32_t sessionId_ = 0; - // the sequenceId as key - std::map reSendMap_; - // remaining sending window - int32_t windowSize_ = 0; - // in this session, the max sequenceId has been sent - uint32_t maxSequenceIdhasSent_ = 0; - // int this session, all data has been sent, but all ack has received or not - bool isAllDataHasSent_ = false; - SingleVerSyncTaskContext *context_ = nullptr; - std::shared_ptr dataSync_ = nullptr; - int mode_ = 0; // sync mode, e.g. push - bool isErr_ = true; -}; -} // namespace DistributedDB -#endif // SLIDING_WINDOW_SENDER_H - diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp index 413d37543..d07877666 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp @@ -93,10 +93,7 @@ int SyncStateMachine::TimeoutCallback(TimerId timerId) } retryTime++; syncContext_->SetRetryTime(retryTime); - // when version is high than 102, the sequenceid will be managed by slide windows sender. - if (syncContext_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - syncContext_->IncSequenceId(); - } + // the sequenceid will be managed by dataSync slide windows. syncContext_->SetRetryStatus(SyncTaskContext::NEED_RETRY); int timeoutTime = syncContext_->GetSyncRetryTimeout(retryTime); syncContext_->ModifyTimer(timeoutTime); @@ -125,12 +122,6 @@ void SyncStateMachine::Abort() int SyncStateMachine::SwitchMachineState(uint8_t event) { - if (syncContext_->GetRemoteSoftwareVersion() == SOFTWARE_VERSION_EARLIEST) { - currentSyncProctolVersion_ = SINGLE_VER_SYNC_PROCTOL_V2; // ver 101 102 use same table - } else if (syncContext_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_EARLIEST) { - currentSyncProctolVersion_ = syncContext_->GetRemoteSoftwareVersion(); - } - const std::vector &tables = GetStateSwitchTables(); auto tableIter = std::find_if(tables.begin(), tables.end(), [this](const StateSwitchTable &table) { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp index 099a76277..909d78c58 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp @@ -109,8 +109,8 @@ void SyncTaskContext::SetOperationStatus(int status) LOGD("[SyncTaskContext][SetStatus] syncOperation is null"); return; } - int finalStatus = status; + int operationStatus = syncOperation_->GetStatus(deviceId_); if (status == SyncOperation::OP_SEND_FINISHED && operationStatus == SyncOperation::OP_RECV_FINISHED) { if (GetTaskErrCode() == -E_EKEYREVOKED) { diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 3ca3c7d54..0f731fedb 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -216,9 +216,9 @@ ohos_source_set("src_file") { "../syncer/src/multi_ver_sync_task_context.cpp", "../syncer/src/multi_ver_syncer.cpp", "../syncer/src/query_sync_water_mark_helper.cpp", + "../syncer/src/single_ver_data_message_schedule.cpp" "../syncer/src/single_ver_data_packet.cpp", "../syncer/src/single_ver_data_sync.cpp", - "../syncer/src/single_ver_data_sync_with_sliding_window.cpp", "../syncer/src/single_ver_kv_syncer.cpp", "../syncer/src/single_ver_relational_syncer.cpp", "../syncer/src/single_ver_serialize_manager.cpp", @@ -227,8 +227,6 @@ ohos_source_set("src_file") { "../syncer/src/single_ver_sync_target.cpp", "../syncer/src/single_ver_sync_task_context.cpp", "../syncer/src/single_ver_syncer.cpp", - "../syncer/src/sliding_window_receiver.cpp", - "../syncer/src/sliding_window_sender.cpp", "../syncer/src/subscribe_manager.cpp", "../syncer/src/sync_engine.cpp", "../syncer/src/sync_operation.cpp", @@ -386,8 +384,8 @@ distributeddb_unittest("DistributedDBSingleVerP2PSyncTest") { [ "unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp" ] } -distributeddb_unittest("DistributedDBSyncModuleTest") { - sources = [ "unittest/common/syncer/distributeddb_sync_module_test.cpp" ] +distributeddb_unittest("DistributedDBSingleVerMsgScheduleTest") { + sources = [ "unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp" ] } distributeddb_unittest("DistributedDBInterfacesNBDelegateTest") { @@ -671,6 +669,7 @@ group("unittest") { ":DistributedDBSingleVerP2PSubscribeSyncTest", ":DistributedDBSingleVerP2PSyncCheckTest", ":DistributedDBSingleVerP2PSyncTest", + ":DistributedDBSingleVerMsgScheduleTest", ":DistributedDBSingleVersionResultSetTest", ":DistributedDBSqliteRegisterTest", ":DistributedDBStorageCommitStorageTest", @@ -685,7 +684,6 @@ group("unittest") { ":DistributedDBStorageSingleVerUpgradeTest", ":DistributedDBStorageTransactionDataTest", ":DistributedDBStorageTransactionRecordTest", - ":DistributedDBSyncModuleTest", ":DistributedDBSyncerDeviceManagerTest", ":DistributedDBTimeSyncTest", ":RuntimeContextProcessSystemApiAdapterImplTest", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp new file mode 100644 index 000000000..cff9bc90c --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "distributeddb_tools_unit_test.h" +#include "single_ver_data_message_schedule.h" +#include "single_ver_data_packet.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { +} + +class DistributedDBSingleVerMsgScheduleTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBSingleVerMsgScheduleTest::SetUpTestCase(void) +{ +} + +void DistributedDBSingleVerMsgScheduleTest::TearDownTestCase(void) +{ +} + +void DistributedDBSingleVerMsgScheduleTest::SetUp(void) +{ +} + +void DistributedDBSingleVerMsgScheduleTest::TearDown(void) +{ +} + +/** + * @tc.name: MsgSchedule001 + * @tc.desc: Test MsgSchedule funtion with normal sequenceId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule001, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg sequence_3, sequence_2, sequence_1 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 3; i >= 1; i--) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + if (i > 1) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + } + } + /** + * @tc.steps: step2. get msg + * @tc.expected: get msg by sequence_1, sequence_2, sequence_3 + */ + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule002 + * @tc.desc: Test MsgSchedule funtion with by low version + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule002, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg session1_sequence1, session2_sequence1 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_RELEASE_2_0); + DataSyncMessageInfo info; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sessionId_ = i; + info.sequenceId_ = 1; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + /** + * @tc.steps: step2. get msg + * @tc.expected: get msg by session2_sequence1 + */ + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), 1u); + msgSchedule.ScheduleInfoHandle(false, false, msg); + delete msg; + msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule003 + * @tc.desc: Test MsgSchedule funtion with cross sessionId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule003, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg session1_seq1, session2_seq1, session1_seq2, session2_seq2, and handle session1_seq1 + * @tc.expected: handle ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + bool isNeedHandle = true; + bool isNeedContinue = true; + info.sessionId_ = 1; + info.sequenceId_ = 1; + info.packetId_ = 1; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + info.sessionId_ = 2; + info.sequenceId_ = 1; + info.packetId_ = 1; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + info.sessionId_ = 1; + info.sequenceId_ = 2; + info.packetId_ = 2; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + info.sessionId_ = 2; + info.sequenceId_ = 2; + info.packetId_ = 2; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + + /** + * @tc.steps: step2. get msg + * @tc.expected: get msg by session2_seq1, session2_seq2 + */ + for (uint32_t i = 1; i <= 2; i++) { + msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + EXPECT_EQ(msg->GetSessionId(), 2u); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule004 + * @tc.desc: Test MsgSchedule funtion with same sessionId with different packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule004, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq2_packet2, seq3_packet3, seq1_packet4, seq2_packet5, seq3_packet6 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 2; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + } + for (uint32_t i = 1; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i + 3; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + /** + * @tc.steps: step2. get msg + * @tc.expected: drop seq2_packet2, seq3_packet3 and get seq1_packet4, seq2_packet5, seq3_packet6 + */ + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i + 3); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule005 + * @tc.desc: Test MsgSchedule funtion with same sessionId with different packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule005, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq1_packet4, seq2_packet5, seq2_packet2, seq3_packet3 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i + 3; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + for (uint32_t i = 2; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + /** + * @tc.steps: step2. get msg + * @tc.expected: drop seq2_packet2, seq3_packet3 and get seq1_packet4, seq2_packet5 + */ + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i + 3); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule006 + * @tc.desc: Test MsgSchedule funtion with same sessionId and same sequenceId and packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule006, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq1_packet1, seq2_packet2 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + for (uint32_t i = 1; i <= 2; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + /** + * @tc.steps: step2. put msg seq1_packet1, seq2_packet2 again and seq3_packet3 + * @tc.expected: get msg ok and get seq1_packet1, seq2_packet2 and seq3_packet3 + */ + for (uint32_t i = 1; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, (i == 3) ? true : false); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule007 + * @tc.desc: Test MsgSchedule funtion with same sessionId and duplicate sequenceId and low packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule007, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq1_packet4, seq2_packet5 + * @tc.expected: put msg ok and get msg seq1_packet4, seq2_packet5 + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i + 3; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + for (uint32_t i = 1; i <= 2; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i + 3); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + /** + * @tc.steps: step2. put msg seq1_packet1, seq2_packet2 + * @tc.expected: get nullptr + */ + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + EXPECT_EQ(isNeedContinue, true); + ASSERT_TRUE(msg == nullptr); + } + RefObject::KillAndDecObjRef(context); + context = nullptr; +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp deleted file mode 100644 index 2fc18aa7e..000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "db_constant.h" -#include "distributeddb_tools_unit_test.h" -#include "mock_communicator.h" -#include "mock_single_ver_data_sync.h" -#include "mock_single_ver_sync_task_context.h" -#include "sliding_window_receiver.h" - -using namespace testing::ext; -using namespace DistributedDB; -using namespace DistributedDBUnitTest; - -class DistributedDBSyncModuleTest : public testing::Test { -public: - static void SetUpTestCase(void); - static void TearDownTestCase(void); - void SetUp(); - void TearDown(); -}; - -void DistributedDBSyncModuleTest::SetUpTestCase(void) -{ -} - -void DistributedDBSyncModuleTest::TearDownTestCase(void) -{ -} - -void DistributedDBSyncModuleTest::SetUp(void) -{ -} - -void DistributedDBSyncModuleTest::TearDown(void) -{ -} - - -namespace { - void InitMessage(DataSyncMessageInfo &info) - { - info.messageId_ = DistributedDB::DATA_SYNC_MESSAGE; - info.messageType_ = DistributedDB::TYPE_REQUEST; - info.sendCode_ = -DistributedDB::E_UNFINISHED; - info.mode_ = DistributedDB::PUSH; - } - - int BuildPushMessage(DataSyncMessageInfo &info, DistributedDB::Message *&message) - { - return DistributedDBToolsUnitTest::BuildMessage(info, message); - } -} - -/** - * @tc.name: SlidingWindow001 - * @tc.desc: Test sliding window receive data when water error happen - * @tc.type: FUNC - * @tc.require: AR000EV1G6 - * @tc.author: zhangqiquan - */ -HWTEST_F(DistributedDBSyncModuleTest, SlidingWindow001, TestSize.Level1) -{ - std::shared_ptr mockSingleVerDataSync = std::make_shared(); - auto *context = new MockSingleVerSyncTaskContext(); - - SlidingWindowReceiver receiver; - receiver.Initialize(context, mockSingleVerDataSync); - uint32_t sequenceId = 1; - uint32_t sessionId = 1; - uint64_t packedId = 1; - DataSyncMessageInfo info{}; - InitMessage(info); - info.sequenceId_ = sequenceId; - info.sessionId_ = sessionId; - info.packetId_ = packedId; - - DistributedDB::Message *message = nullptr; - int status = BuildPushMessage(info, message); - EXPECT_EQ(status, E_OK); - - /** - * @tc.steps: step1. sliding window receive data first and now water error happen - * @tc.expected: step1. receive msg should return -E_NOT_NEED_DELETE_MSG. - */ - EXPECT_CALL(*context, HandleDataRequestRecv(testing::_)).WillRepeatedly(testing::Return(E_OK)); - EXPECT_CALL(*static_cast(mockSingleVerDataSync.get()), - SendDataAck(testing::_, testing::_, testing::_, testing::_)).WillRepeatedly(testing::Return(E_OK)); - context->SetReceiveWaterMarkErr(true); - EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); - - /** - * @tc.steps: step2. sliding window receive data which is has diff sessionId - * @tc.expected: step2. receive msg should return -E_NOT_NEED_DELETE_MSG. - */ - info.sessionId_++; - status = BuildPushMessage(info, message); - EXPECT_EQ(status, E_OK); - EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); - - /** - * @tc.steps: step3. sliding window receive data which is has diff sequenceId - * @tc.expected: step3. receive msg should return -E_NOT_NEED_DELETE_MSG. - */ - info.sequenceId_++; - info.packetId_++; - status = BuildPushMessage(info, message); - EXPECT_EQ(status, E_OK); - EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); - - receiver.Clear(); - // ensure to call DecRefObj in another thread - std::this_thread::sleep_for(std::chrono::seconds(1)); - RefObject::KillAndDecObjRef(context); - context = nullptr; -} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h deleted file mode 100644 index 5bd453f60..000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MOCK_SINGLE_VER_DATA_SYNC_H -#define MOCK_SINGLE_VER_DATA_SYNC_H - -#include -#include "single_ver_data_sync.h" - -namespace DistributedDB { -class MockSingleVerDataSync : public SingleVerDataSync { -public: - MOCK_METHOD4(SendDataAck, int(SingleVerSyncTaskContext *, const Message *, int32_t, WaterMark)); -}; -} // namespace DistributedDB - -#endif // MOCK_SINGLE_VER_DATA_SYNC_H diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h deleted file mode 100644 index 23ad7d58a..000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H -#define MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H - -#include -#include "single_ver_sync_task_context.h" - -namespace DistributedDB { -class MockSingleVerSyncTaskContext : public SingleVerSyncTaskContext { -public: - MOCK_METHOD1(HandleDataRequestRecv, int(const Message *)); - MOCK_CONST_METHOD0(IsReceiveWaterMarkErr, bool(void)); -}; -} // namespace DistributedDB -#endif // MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H -- Gitee From 84d27c8846fdad485bb4a46e200a70efe8442101 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Tue, 4 Jan 2022 19:01:40 +0800 Subject: [PATCH 23/53] Fix reviews Signed-off-by: lianhuix --- .../libs/distributeddb/common/src/json_object.cpp | 3 +++ .../common/src/relational/relational_schema_object.cpp | 2 +- .../interfaces/src/relational/data_value.cpp | 8 ++++---- .../src/relational/relational_store_manager.cpp | 3 +-- .../storage/src/relational_sync_able_storage.cpp | 5 ++++- .../src/sqlite/relational/sqlite_relational_store.cpp | 6 +++--- 6 files changed, 16 insertions(+), 11 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index be8eab8ea..5a4d0a01a 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -727,6 +727,7 @@ int JsonObject::GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector -#include #include "json_object.h" #include "relational_schema_object.h" +#include "schema_utils.h" namespace DistributedDB { const std::string &FieldInfo::GetFieldName() const diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index ad5072181..1aa170869 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -46,7 +46,7 @@ uint32_t Blob::GetSize() const int Blob::WriteBlob(const uint8_t *ptrArray, const uint32_t &size) { if (size == 0) return E_OK; - ptr_ = new(std::nothrow) uint8_t[size]; + ptr_ = new (std::nothrow) uint8_t[size]; if (ptr_ == nullptr) { return -E_OUT_OF_MEMORY; } @@ -159,7 +159,7 @@ DataValue &DataValue::operator=(const Blob &blob) if (blob.GetSize() <= 0) { return *this; } - value_.blobPtr = new(std::nothrow) Blob(); + value_.blobPtr = new (std::nothrow) Blob(); if (value_.blobPtr == nullptr) { return *this; } @@ -173,7 +173,7 @@ DataValue &DataValue::operator=(const Blob &blob) DataValue &DataValue::operator=(const std::string &string) { ResetValue(); - value_.blobPtr = new(std::nothrow) Blob(); + value_.blobPtr = new (std::nothrow) Blob(); if (value_.blobPtr == nullptr) { return *this; } @@ -247,7 +247,7 @@ int DataValue::GetBlob(Blob *&outVal) const if (type_ != StorageType::STORAGE_TYPE_BLOB && type_ != StorageType::STORAGE_TYPE_TEXT) { return -E_NOT_SUPPORT; } - outVal = new(std::nothrow) Blob(); + outVal = new (std::nothrow) Blob(); if (outVal == nullptr) { return -E_OUT_OF_MEMORY; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index e182aa33a..593efda1a 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -80,9 +80,8 @@ DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const int errCode = E_OK; auto *conn = GetOneConnectionWithRetry(properties, errCode); - DBStatus status = TransferDBErrno(errCode); if (conn == nullptr) { - return status; + return TransferDBErrno(errCode); } delegate = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index 2cc4aab84..2933959e9 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -24,6 +24,10 @@ namespace DistributedDB { } \ } while (0) +namespace { +static constexpr float QUERY_SYNC_THRESHOLD = 0.50; +} + RelationalSyncAbleStorage::RelationalSyncAbleStorage(StorageEngine *engine) : storageEngine_(static_cast(engine)) {} @@ -224,7 +228,6 @@ static size_t GetDataItemSerialSize(const DataItem &item, size_t appendLen) return dataSize; } -static constexpr float QUERY_SYNC_THRESHOLD = 0.50; static bool CanHoldDeletedData(const std::vector &dataItems, const DataSizeSpecInfo &dataSizeInfo, size_t appendLen) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 5a8a845b1..c33c73acd 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -174,7 +174,7 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } - storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); + storageEngine_ = new (std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); if (storageEngine_ == nullptr) { LOGE("[RelationalStore][Open] Create syncable storage failed"); errCode = -E_OUT_OF_MEMORY; @@ -311,7 +311,7 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) std::lock_guard lock(schemaMutex_); auto schema = properties_.GetSchema(); if (schema.GetTable(tableName).GetTableName() == tableName) { - LOGW("distributed table %s was already created.", tableName.c_str()); + LOGW("distributed table was already created."); return E_OK; } @@ -320,7 +320,7 @@ int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) return -E_MAX_LIMITS; } - LOGD("Create distributed table for %s.", tableName.c_str()); + LOGD("Create distributed table."); auto *handle = GetHandle(true, errCode); if (handle == nullptr) { return errCode; -- Gitee From 72bb79bed294c2a5e444b34d9548815e4f712e37 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Fri, 31 Dec 2021 10:17:33 +0800 Subject: [PATCH 24/53] Change check table is empty with min(rowid) Signed-off-by: lianhuix --- ...single_ver_relational_storage_executor.cpp | 20 ++++++++++++------- .../storage/src/sqlite/sqlite_utils.cpp | 10 +++++++--- .../storage/src/sqlite/sqlite_utils.h | 2 +- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index bb2d52f11..c704a5c1d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -29,23 +29,29 @@ int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std:: return -E_INVALID_DB; } - int historyDataCnt = 0; - int errCode = SQLiteUtils::GetTableRecordCount(dbHandle_, tableName, historyDataCnt); + int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode != E_OK) { - LOGE("[CreateDistributedTable] Get the number of table [%s] rows failed. %d", tableName.c_str(), errCode); + LOGE("[CreateDistributedTable] analysis table schema failed. %d", errCode); return errCode; } - if (historyDataCnt > 0) { // 0 : create distributed table should on an empty table - LOGE("[CreateDistributedTable] Create distributed table should on an empty table."); + if (table.GetCreateTableSql().find("WITHOUT ROWID") != std::string::npos) { + LOGE("[CreateDistributedTable] Not support create distributed table without rowid."); return -E_NOT_SUPPORT; } - errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); + bool isTableEmpty = false; + errCode = SQLiteUtils::CheckTableEmpty(dbHandle_, tableName, isTableEmpty); if (errCode != E_OK) { - LOGE("[CreateDistributedTable] analysis table schema failed"); + LOGE("[CreateDistributedTable] Check table [%s] is empty failed. %d", tableName.c_str(), errCode); return errCode; } + + if (!isTableEmpty) { // create distributed table should on an empty table + LOGE("[CreateDistributedTable] Create distributed table should on an empty table."); + return -E_NOT_SUPPORT; + } + // create log table errCode = SQLiteUtils::CreateRelationalLogTable(dbHandle_, tableName); if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index e69117af7..88cabd62e 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1974,13 +1974,13 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) return E_OK; } -int SQLiteUtils::GetTableRecordCount(sqlite3 *db, const std::string &tableName, int &count) +int SQLiteUtils::CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool &isEmpty) { if (db == nullptr) { return -E_INVALID_ARGS; } - std::string cntSql = "SELECT COUNT(*) FROM " + tableName + ";"; + std::string cntSql = "SELECT min(rowid) FROM " + tableName + ";"; sqlite3_stmt *stmt = nullptr; int errCode = SQLiteUtils::GetStatement(db, cntSql, stmt); if (errCode != E_OK) { @@ -1989,7 +1989,11 @@ int SQLiteUtils::GetTableRecordCount(sqlite3 *db, const std::string &tableName, errCode = SQLiteUtils::StepWithRetry(stmt, false); if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - count = sqlite3_column_int(stmt, 0); + if (sqlite3_column_type(stmt, 0) == SQLITE_NULL) { + isEmpty = true; + } else { + isEmpty = false; + } errCode = E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index c839ca036..1c6a1ff8a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -177,7 +177,7 @@ public: static int ExpandedSql(sqlite3_stmt *stmt, std::string &basicString); - static int GetTableRecordCount(sqlite3 *db, const std::string &tableName, int &count); + static int CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool &isEmpty); private: -- Gitee From d9285191da51ad2deb848837aeb968ad7616aff1 Mon Sep 17 00:00:00 2001 From: zqq Date: Thu, 30 Dec 2021 20:31:58 +0800 Subject: [PATCH 25/53] 1.Add block sync / Async interface and implement 2.Add set autoLaunch callback interface Signed-off-by: zqq --- .../include/auto_launch_export.h | 1 + .../relational/relational_store_delegate.h | 3 + .../relational/relational_store_manager.h | 5 + .../relational_store_delegate_impl.cpp | 59 +++- .../relational_store_delegate_impl.h | 6 + .../relational/relational_store_manager.cpp | 15 + .../include/relational_store_connection.h | 2 +- .../src/single_ver_relational_syncer.cpp | 305 +++++++++--------- .../syncer/src/single_ver_relational_syncer.h | 6 +- 9 files changed, 245 insertions(+), 157 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h index ea7376a1a..4fba67456 100755 --- a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h +++ b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h @@ -46,6 +46,7 @@ struct AutoLaunchParam { std::string storeId; AutoLaunchOption option; AutoLaunchNotifier notifier; + std::string path; }; using AutoLaunchRequestCallback = std::function; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 1b7f3d3ef..763d6354c 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -40,6 +40,9 @@ public: SyncStatusCallback &onComplete, bool wait) = 0; DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, + const Query &query, bool wait, std::map> &devicesMap) = 0; + + DB_API virtual DBStatus ASync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, const Query &query, bool wait) = 0; DB_API virtual DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 19642b597..1fed6cbd2 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -41,6 +41,11 @@ public: DB_API DBStatus DeleteStore(const std::string &path); + DB_API static void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback); + + DB_API static std::string GetRelationalStoreIdentifier(const std::string &userId, const std::string &appId, + const std::string &storeId); + private: std::string appId_; std::string userId_; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index c9c1b842e..a1f4a750b 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -74,6 +74,24 @@ DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string & } DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, + const Query &query, bool wait, std::map> &devicesMap) +{ + bool syncFinished = false; + std::condition_variable cv; + SyncStatusCallback onComplete = [&syncFinished, &cv, &devicesMap]( + const std::map> &resMap) { + syncFinished = true; + devicesMap = resMap; + cv.notify_all(); + }; + DBStatus errCode = ASync(devices, mode, onComplete, query, wait); + std::mutex mutex; + std::unique_lock lock(mutex); + cv.wait(lock, [&syncFinished]{ return syncFinished; }); + return errCode; +} + +DBStatus RelationalStoreDelegateImpl::ASync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, const Query &query, bool wait) { if (conn_ == nullptr) { @@ -81,7 +99,8 @@ DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devic return DB_ERROR; } - RelationalStoreConnection::SyncInfo syncInfo{devices, mode, onComplete, query, wait}; + RelationalStoreConnection::SyncInfo syncInfo{devices, mode, + std::bind(&RelationalStoreDelegateImpl::OnSyncComplete, std::placeholders::_1, onComplete), query, wait}; int errCode = conn_->SyncToDevice(syncInfo); if (errCode != E_OK) { LOGW("[RelationalStore Delegate] sync data to device failed:%d", errCode); @@ -120,5 +139,43 @@ void RelationalStoreDelegateImpl::SetReleaseFlag(bool flag) { releaseFlag_ = flag; } + +void RelationalStoreDelegateImpl::OnSyncComplete(const std::map> &devicesMap, + SyncStatusCallback &onComplete) +{ + static const std::map statusMap = { + { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, + { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, + { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, + { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, + { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, + { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, + { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, + { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, + { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, + { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, + { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, + { static_cast(SyncOperation::OP_INTERCEPT_DATA_FAIL), INTERCEPT_DATA_FAIL }, + { static_cast(SyncOperation::OP_MAX_LIMITS), OVER_MAX_LIMITS }, + { static_cast(SyncOperation::OP_INVALID_ARGS), INVALID_ARGS }, + }; + std::map> res; + for (const auto &[device, statusList] : devicesMap) { + for (const auto &tableStatus : statusList) { + TableStatus table; + table.tableName = tableStatus.tableName; + DBStatus status = DB_ERROR; + auto iterator = statusMap.find(tableStatus.status); + if (iterator != statusMap.end()) { + status = iterator->second; + } + table.status = status; + res[device].push_back(table); + } + } + if (onComplete) { + onComplete(res); + } +} } // namespace DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 25d7e6c96..4ca8de9ae 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -35,6 +35,9 @@ public: SyncStatusCallback &onComplete, bool wait) override; DBStatus Sync(const std::vector &devices, SyncMode mode, + const Query &query, bool wait, std::map> &devicesMap) override; + + DBStatus ASync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, const Query &query, bool wait) override; DBStatus RemoveDeviceData(const std::string &device) override; @@ -48,6 +51,9 @@ public: void SetReleaseFlag(bool flag); private: + static void OnSyncComplete(const std::map> &devicesMap, + SyncStatusCallback &onComplete); + RelationalStoreConnection *conn_ = nullptr; std::string storePath_; std::atomic releaseFlag_ = false; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 593efda1a..d2a3bd5cc 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -17,6 +17,7 @@ #include +#include "auto_launch.h" #include "relational_store_instance.h" #include "db_common.h" #include "param_check_utils.h" @@ -24,6 +25,7 @@ #include "db_errno.h" #include "kv_store_errno.h" #include "relational_store_delegate_impl.h" +#include "runtime_context.h" #include "platform_specific.h" namespace DistributedDB { @@ -170,5 +172,18 @@ DBStatus RelationalStoreManager::DeleteStore(const std::string &path) LOGE("Delete the kv store error:%d", errCode); return TransferDBErrno(errCode); } + +void RelationalStoreManager::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) +{ +} + +std::string RelationalStoreManager::GetRelationalStoreIdentifier(const std::string &userId, const std::string &appId, + const std::string &storeId) +{ + if (!ParamCheckUtils::CheckStoreParameter(storeId, appId, userId)) { + return ""; + } + return DBCommon::TransferHashString(userId + "-" + appId + "-" + storeId); +} } // namespace DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index ac6cb4c13..2ffed77c2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -30,7 +30,7 @@ public: struct SyncInfo { const std::vector &devices; SyncMode mode = SYNC_MODE_PUSH_PULL; - SyncStatusCallback &onComplete; + const SyncStatusCallback &onComplete; const Query &query; bool wait = true; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp index 31fc0d114..df15a8dab 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp @@ -1,154 +1,153 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "single_ver_relational_syncer.h" -#ifdef RELATIONAL_STORE -#include "db_common.h" -#include "relational_db_sync_interface.h" -#include "single_ver_sync_engine.h" - -namespace DistributedDB { -int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) -{ - const auto &syncInterface = static_cast(syncInterface_); - std::vector tablesQuery; - if (param.isQuerySync) { - tablesQuery.push_back(param.syncQuery); - } else { - tablesQuery = syncInterface->GetTablesQuery(); - } - std::set subSyncIdSet; - int errCode = GenerateEachSyncTask(param, syncId, tablesQuery, subSyncIdSet); - if (errCode != E_OK) { - DoRollBack(subSyncIdSet); - return errCode; - } - return E_OK; -} - -int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, - const std::vector &tablesQuery, std::set &subSyncIdSet) -{ - SyncParma subParam = param; - subParam.isQuerySync = true; - int errCode = E_OK; - for (const QuerySyncObject &table : tablesQuery) { - uint32_t subSyncId = GenerateSyncId(); - LOGI("[SingleVerRelationalSyncer] SubSyncId %d create by SyncId %d, tableName = %s", - subSyncId, syncId, table.GetRelationTableName().c_str()); - subParam.syncQuery = table; - subParam.onComplete = std::bind(&SingleVerRelationalSyncer::DoOnSubSyncComplete, this, subSyncId, - syncId, param, std::placeholders::_1); - { - std::lock_guard lockGuard(syncMapLock_); - fullSyncIdMap_[syncId].insert(subSyncId); - } - errCode = GenericSyncer::PrepareSync(subParam, subSyncId); - if (errCode != E_OK) { - LOGW("[SingleVerRelationalSyncer] PrepareSync failed errCode:%d", errCode); - std::lock_guard lockGuard(syncMapLock_); - fullSyncIdMap_[syncId].erase(subSyncId); - break; - } - subSyncIdSet.insert(subSyncId); - } - return errCode; -} - -void SingleVerRelationalSyncer::DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, - const SyncParma ¶m, const std::map &devicesMap) -{ - static std::map statusMap = { - { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, - { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, - { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, - { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, - { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, - { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, - { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, - { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, - { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, - { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, - { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, - }; - bool allFinish = true; - { - std::lock_guard lockGuard(syncMapLock_); - fullSyncIdMap_[syncId].erase(subSyncId); - allFinish = fullSyncIdMap_[syncId].empty(); - for (const auto &item : devicesMap) { - DBStatus status = DB_ERROR; - if (statusMap.find(item.second) != statusMap.end()) { - status = statusMap[item.second]; - } - resMap_[syncId][item.first][param.syncQuery.GetRelationTableName()] = status; - } - } - if (allFinish) { - DoOnComplete(param, syncId); - } -} - -void SingleVerRelationalSyncer::DoRollBack(std::set &subSyncIdSet) -{ - for (const auto &removeId : subSyncIdSet) { - int retCode = RemoveSyncOperation((int)removeId); - if (retCode != E_OK) { - LOGW("[SingleVerRelationalSyncer] RemoveSyncOperation failed errCode:%d, syncId:%d", retCode, removeId); - } - } -} - -void SingleVerRelationalSyncer::DoOnComplete(const SyncParma ¶m, uint32_t syncId) -{ - if (!param.relationOnComplete) { - return; - } - std::map> syncRes; - std::map> tmpMap; - { - std::lock_guard lockGuard(syncMapLock_); - tmpMap = resMap_[syncId]; - } - for (const auto &devicesRes : tmpMap) { - for (const auto &tableRes : devicesRes.second) { - syncRes[devicesRes.first].push_back( - {tableRes.first, static_cast(tableRes.second)}); - } - } - param.relationOnComplete(syncRes); - { - std::lock_guard lockGuard(syncMapLock_); - resMap_.erase(syncId); - } -} - -void SingleVerRelationalSyncer::EnableAutoSync(bool enable) -{ -} - -int SingleVerRelationalSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) -{ - return E_OK; -} - -void SingleVerRelationalSyncer::LocalDataChanged(int notifyEvent) -{ -} - -void SingleVerRelationalSyncer::RemoteDataChanged(const std::string &device) -{ -} -} +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "single_ver_relational_syncer.h" +#ifdef RELATIONAL_STORE +#include "db_common.h" +#include "relational_db_sync_interface.h" +#include "single_ver_sync_engine.h" + +namespace DistributedDB { +int SingleVerRelationalSyncer::Sync(const SyncParma ¶m) +{ + if (param.mode == SYNC_MODE_PUSH_PULL) { + return -E_NOT_SUPPORT; + } + if (param.syncQuery.GetRelationTableName().empty()) { + return -E_NOT_SUPPORT; + } + return GenericSyncer::Sync(param); +} + +int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) +{ + const auto &syncInterface = static_cast(syncInterface_); + std::vector tablesQuery; + if (param.isQuerySync) { + tablesQuery.push_back(param.syncQuery); + } else { + tablesQuery = syncInterface->GetTablesQuery(); + } + std::set subSyncIdSet; + int errCode = GenerateEachSyncTask(param, syncId, tablesQuery, subSyncIdSet); + if (errCode != E_OK) { + DoRollBack(subSyncIdSet); + return errCode; + } + if (param.wait) { + DoOnComplete(param, syncId); + } + return E_OK; +} + +int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, + const std::vector &tablesQuery, std::set &subSyncIdSet) +{ + SyncParma subParam = param; + subParam.isQuerySync = true; + int errCode = E_OK; + for (const QuerySyncObject &table : tablesQuery) { + uint32_t subSyncId = GenerateSyncId(); + LOGI("[SingleVerRelationalSyncer] SubSyncId %d create by SyncId %d, tableName = %s", + subSyncId, syncId, table.GetRelationTableName().c_str()); + subParam.syncQuery = table; + subParam.onComplete = std::bind(&SingleVerRelationalSyncer::DoOnSubSyncComplete, this, subSyncId, + syncId, param, std::placeholders::_1); + { + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].insert(subSyncId); + } + errCode = GenericSyncer::PrepareSync(subParam, subSyncId); + if (errCode != E_OK) { + LOGW("[SingleVerRelationalSyncer] PrepareSync failed errCode:%d", errCode); + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].erase(subSyncId); + break; + } + subSyncIdSet.insert(subSyncId); + } + return errCode; +} + +void SingleVerRelationalSyncer::DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, + const SyncParma ¶m, const std::map &devicesMap) +{ + bool allFinish = true; + { + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].erase(subSyncId); + allFinish = fullSyncIdMap_[syncId].empty(); + for (const auto &item : devicesMap) { + resMap_[syncId][item.first][param.syncQuery.GetRelationTableName()] = static_cast(item.second); + } + } + // block sync do callback in sync function + if (allFinish && !param.wait) { + DoOnComplete(param, syncId); + } +} + +void SingleVerRelationalSyncer::DoRollBack(std::set &subSyncIdSet) +{ + for (const auto &removeId : subSyncIdSet) { + int retCode = RemoveSyncOperation((int)removeId); + if (retCode != E_OK) { + LOGW("[SingleVerRelationalSyncer] RemoveSyncOperation failed errCode:%d, syncId:%d", retCode, removeId); + } + } +} + +void SingleVerRelationalSyncer::DoOnComplete(const SyncParma ¶m, uint32_t syncId) +{ + if (!param.relationOnComplete) { + return; + } + std::map> syncRes; + std::map> tmpMap; + { + std::lock_guard lockGuard(syncMapLock_); + tmpMap = resMap_[syncId]; + } + for (const auto &devicesRes : tmpMap) { + for (const auto &tableRes : devicesRes.second) { + syncRes[devicesRes.first].push_back( + {tableRes.first, static_cast(tableRes.second)}); + } + } + param.relationOnComplete(syncRes); + { + std::lock_guard lockGuard(syncMapLock_); + resMap_.erase(syncId); + fullSyncIdMap_.erase(syncId); + } +} + +void SingleVerRelationalSyncer::EnableAutoSync(bool enable) +{ +} + +int SingleVerRelationalSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +{ + return E_OK; +} + +void SingleVerRelationalSyncer::LocalDataChanged(int notifyEvent) +{ +} + +void SingleVerRelationalSyncer::RemoteDataChanged(const std::string &device) +{ +} +} #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h index 159a011ea..15e4ef814 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h @@ -22,6 +22,9 @@ public: SingleVerRelationalSyncer() = default; ~SingleVerRelationalSyncer() override = default; + // Sync function. use SyncParma to reduce paramter. + int Sync(const SyncParma ¶m) override; + void EnableAutoSync(bool enable) override; int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; @@ -50,6 +53,5 @@ private: std::map>> resMap_; }; } - #endif -#endif // RELATIONAL_SYNCER_H +#endif // RELATIONAL_SYNCER_H \ No newline at end of file -- Gitee From a163369d81d51b6a6bd5c1fca71c157b2315476f Mon Sep 17 00:00:00 2001 From: zqq Date: Thu, 30 Dec 2021 20:48:51 +0800 Subject: [PATCH 26/53] 1.Add block sync / Async interface and implement 2.Add set autoLaunch callback interface Signed-off-by: zqq --- .../relational/relational_store_delegate.h | 4 ++-- .../relational_store_delegate_impl.cpp | 21 +++++++++---------- .../relational_store_delegate_impl.h | 7 +++++-- ...ributeddb_relational_ver_p2p_sync_test.cpp | 15 +------------ 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 763d6354c..65d2054da 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -40,10 +40,10 @@ public: SyncStatusCallback &onComplete, bool wait) = 0; DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, - const Query &query, bool wait, std::map> &devicesMap) = 0; + const Query &query, std::map> &devicesMap) = 0; DB_API virtual DBStatus ASync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait) = 0; + SyncStatusCallback &onComplete, const Query &query) = 0; DB_API virtual DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) = 0; }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index a1f4a750b..f2218ea61 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -74,24 +74,22 @@ DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string & } DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, - const Query &query, bool wait, std::map> &devicesMap) + const Query &query, std::map> &devicesMap) { - bool syncFinished = false; - std::condition_variable cv; - SyncStatusCallback onComplete = [&syncFinished, &cv, &devicesMap]( + SyncStatusCallback onComplete = [&devicesMap]( const std::map> &resMap) { - syncFinished = true; devicesMap = resMap; - cv.notify_all(); }; - DBStatus errCode = ASync(devices, mode, onComplete, query, wait); - std::mutex mutex; - std::unique_lock lock(mutex); - cv.wait(lock, [&syncFinished]{ return syncFinished; }); - return errCode; + return Sync(devices, mode, onComplete, query, true); } DBStatus RelationalStoreDelegateImpl::ASync(const std::vector &devices, SyncMode mode, + SyncStatusCallback &onComplete, const Query &query) +{ + return Sync(devices, mode, onComplete, query, false); +} + +DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, const Query &query, bool wait) { if (conn_ == nullptr) { @@ -109,6 +107,7 @@ DBStatus RelationalStoreDelegateImpl::ASync(const std::vector &devi return OK; } + DBStatus RelationalStoreDelegateImpl::RemoveDevicesData(const std::string &tableName, const std::string &device) { return NOT_SUPPORT; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 4ca8de9ae..c3173704a 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -35,10 +35,10 @@ public: SyncStatusCallback &onComplete, bool wait) override; DBStatus Sync(const std::vector &devices, SyncMode mode, - const Query &query, bool wait, std::map> &devicesMap) override; + const Query &query, std::map> &devicesMap) override; DBStatus ASync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait) override; + SyncStatusCallback &onComplete, const Query &query) override; DBStatus RemoveDeviceData(const std::string &device) override; @@ -54,6 +54,9 @@ private: static void OnSyncComplete(const std::map> &devicesMap, SyncStatusCallback &onComplete); + DBStatus Sync(const std::vector &devices, SyncMode mode, + SyncStatusCallback &onComplete, const Query &query, bool wait); + RelationalStoreConnection *conn_ = nullptr; std::string storePath_; std::atomic releaseFlag_ = false; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index 1f76b576f..5109d464b 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -148,21 +148,8 @@ namespace { { std::vector devices = {DEVICE_B}; Query query = Query::Select(g_tableName); - std::mutex syncLock; - std::condition_variable syncCondVar; std::map> statusMap; - std::mutex syncMutex; - std::condition_variable syncCv; - SyncStatusCallback callBack = [&statusMap, &syncCv]( - const std::map> &devicesMap) { - statusMap = devicesMap; - syncCv.notify_one(); - }; - DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, callBack, query, false); - std::unique_lock uniqueLock(syncMutex); - syncCv.wait(uniqueLock, [&statusMap]() { - return !statusMap.empty(); - }); + DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, query, statusMap); EXPECT_EQ(callStatus, OK); for (const auto &tablesRes : statusMap) { for (const auto &tableStatus : tablesRes.second) { -- Gitee From 9c27f3fc282a43b8f8c4f5c8e9373c9eef2613ae Mon Sep 17 00:00:00 2001 From: zqq Date: Fri, 31 Dec 2021 09:31:40 +0800 Subject: [PATCH 27/53] Modify rdb sync test case Signed-off-by: zqq --- .../syncer/distributeddb_relational_ver_p2p_sync_test.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index 5109d464b..963983bbe 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -61,6 +61,7 @@ namespace { } EXPECT_EQ(SQLiteUtils::RegisterCalcHash(db), E_OK); EXPECT_EQ(SQLiteUtils::RegisterGetSysTime(db), E_OK); + EXPECT_EQ(sqlite3_exec(db, "PRAGMA journal_mode=WAL;", nullptr, nullptr, nullptr), SQLITE_OK); return rc; } @@ -217,6 +218,9 @@ void DistributedDBRelationalVerP2PSyncTest::SetUpTestCase() */ DistributedDBToolsUnitTest::TestDirInit(g_testDir); g_dbDir = g_testDir + "/test.db"; + sqlite3 *db = nullptr; + ASSERT_EQ(GetDB(db), SQLITE_OK); + sqlite3_close(db); g_communicatorAggregator = new (std::nothrow) VirtualCommunicatorAggregator(); ASSERT_TRUE(g_communicatorAggregator != nullptr); -- Gitee From 89e854c79991c707464498faa1080188510e9cdd Mon Sep 17 00:00:00 2001 From: zqq Date: Fri, 31 Dec 2021 09:54:14 +0800 Subject: [PATCH 28/53] remove empty line Signed-off-by: zqq --- .../interfaces/src/relational/relational_store_delegate_impl.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index f2218ea61..4734789bc 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -107,7 +107,6 @@ DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devic return OK; } - DBStatus RelationalStoreDelegateImpl::RemoveDevicesData(const std::string &tableName, const std::string &device) { return NOT_SUPPORT; -- Gitee From c2c34bfd85c951b44a4d1173d5f75d82f7f2472a Mon Sep 17 00:00:00 2001 From: zqq Date: Fri, 31 Dec 2021 10:46:39 +0800 Subject: [PATCH 29/53] Modify param sequence Signed-off-by: zqq --- .../interfaces/include/relational/relational_store_delegate.h | 2 +- .../src/relational/relational_store_delegate_impl.cpp | 2 +- .../interfaces/src/relational/relational_store_delegate_impl.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 65d2054da..4e71c6d27 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -43,7 +43,7 @@ public: const Query &query, std::map> &devicesMap) = 0; DB_API virtual DBStatus ASync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query) = 0; + const Query &query, SyncStatusCallback &onComplete) = 0; DB_API virtual DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) = 0; }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index 4734789bc..7bfba2e7d 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -84,7 +84,7 @@ DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devic } DBStatus RelationalStoreDelegateImpl::ASync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query) + const Query &query, SyncStatusCallback &onComplete) { return Sync(devices, mode, onComplete, query, false); } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index c3173704a..4f677d24d 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -38,7 +38,7 @@ public: const Query &query, std::map> &devicesMap) override; DBStatus ASync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query) override; + const Query &query, SyncStatusCallback &onComplete) override; DBStatus RemoveDeviceData(const std::string &device) override; -- Gitee From 32306e410b58e576dc6adaa65eb7a22ea7e65ebf Mon Sep 17 00:00:00 2001 From: zqq Date: Fri, 31 Dec 2021 11:00:32 +0800 Subject: [PATCH 30/53] Add lifeCycleCallback interface in rdb connection Signed-off-by: zqq --- .../storage/include/relational_store_connection.h | 5 ++++- .../sqlite/relational/sqlite_relational_store_connection.cpp | 5 +++++ .../sqlite/relational/sqlite_relational_store_connection.h | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index 2ffed77c2..7b84fc277 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -16,10 +16,12 @@ #define RELATIONAL_STORE_CONNECTION_H #ifdef RELATIONAL_STORE +#include "relational_store_delegate.h" #include #include + +#include "db_types.h" #include "macro_utils.h" -#include "relational_store_delegate.h" #include "ref_object.h" namespace DistributedDB { @@ -49,6 +51,7 @@ public: virtual int SyncToDevice(SyncInfo &info) = 0; virtual std::string GetIdentifier() = 0; virtual int CreateDistributedTable(const std::string &tableName) = 0; + virtual int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) = 0; protected: // Get the stashed 'KvDB_ pointer' without ref. diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index 160a64d91..de529540b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -187,5 +187,10 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) } return E_OK; } + +int SQLiteRelationalStoreConnection::RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) +{ + return E_OK; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 63a533f5f..552447e76 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -38,6 +38,7 @@ public: int SyncToDevice(SyncInfo &info) override; std::string GetIdentifier() override; int CreateDistributedTable(const std::string &tableName) override; + int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) override; protected: -- Gitee From 02bcebdbdda0bc68bf03448fa777159d71b71609 Mon Sep 17 00:00:00 2001 From: zqq Date: Wed, 5 Jan 2022 11:48:28 +0800 Subject: [PATCH 31/53] Add rdb sync interface Signed-off-by: zqq --- .../include/relational/relational_store_delegate.h | 3 +++ .../src/relational/relational_store_delegate_impl.cpp | 6 +++--- .../src/relational/relational_store_delegate_impl.h | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 4e71c6d27..d4a8a310c 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -38,6 +38,9 @@ public: DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) = 0; + + DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, + const Query &query, SyncStatusCallback &onComplete, bool wait); DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, const Query &query, std::map> &devicesMap) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index 7bfba2e7d..11fb35067 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -80,17 +80,17 @@ DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devic const std::map> &resMap) { devicesMap = resMap; }; - return Sync(devices, mode, onComplete, query, true); + return Sync(devices, mode, query, onComplete, true); } DBStatus RelationalStoreDelegateImpl::ASync(const std::vector &devices, SyncMode mode, const Query &query, SyncStatusCallback &onComplete) { - return Sync(devices, mode, onComplete, query, false); + return Sync(devices, mode, query, onComplete, false); } DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait) + const Query &query, SyncStatusCallback &onComplete, bool wait) { if (conn_ == nullptr) { LOGE("Invalid connection for operation!"); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 4f677d24d..a190e3316 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -34,6 +34,9 @@ public: DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) override; + DBStatus Sync(const std::vector &devices, SyncMode mode, + const Query &query, SyncStatusCallback &onComplete, bool wait) override; + DBStatus Sync(const std::vector &devices, SyncMode mode, const Query &query, std::map> &devicesMap) override; @@ -54,9 +57,6 @@ private: static void OnSyncComplete(const std::map> &devicesMap, SyncStatusCallback &onComplete); - DBStatus Sync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait); - RelationalStoreConnection *conn_ = nullptr; std::string storePath_; std::atomic releaseFlag_ = false; -- Gitee From c1e0afb2393b945fda3a4cbacbd78a522414a9da Mon Sep 17 00:00:00 2001 From: zqq Date: Wed, 5 Jan 2022 14:09:46 +0800 Subject: [PATCH 32/53] remove unnecessary interface Signed-off-by: zqq --- .../relational/relational_store_delegate.h | 8 +------- .../relational_store_delegate_impl.cpp | 16 ---------------- .../relational/relational_store_delegate_impl.h | 6 ------ 3 files changed, 1 insertion(+), 29 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index d4a8a310c..fbf038564 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -38,15 +38,9 @@ public: DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) = 0; - - DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, - const Query &query, SyncStatusCallback &onComplete, bool wait); DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, - const Query &query, std::map> &devicesMap) = 0; - - DB_API virtual DBStatus ASync(const std::vector &devices, SyncMode mode, - const Query &query, SyncStatusCallback &onComplete) = 0; + const Query &query, SyncStatusCallback &onComplete, bool wait) = 0; DB_API virtual DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) = 0; }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index 11fb35067..5ad622de2 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -73,22 +73,6 @@ DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string & return OK; } -DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, - const Query &query, std::map> &devicesMap) -{ - SyncStatusCallback onComplete = [&devicesMap]( - const std::map> &resMap) { - devicesMap = resMap; - }; - return Sync(devices, mode, query, onComplete, true); -} - -DBStatus RelationalStoreDelegateImpl::ASync(const std::vector &devices, SyncMode mode, - const Query &query, SyncStatusCallback &onComplete) -{ - return Sync(devices, mode, query, onComplete, false); -} - DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, const Query &query, SyncStatusCallback &onComplete, bool wait) { diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index a190e3316..19911f74b 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -37,12 +37,6 @@ public: DBStatus Sync(const std::vector &devices, SyncMode mode, const Query &query, SyncStatusCallback &onComplete, bool wait) override; - DBStatus Sync(const std::vector &devices, SyncMode mode, - const Query &query, std::map> &devicesMap) override; - - DBStatus ASync(const std::vector &devices, SyncMode mode, - const Query &query, SyncStatusCallback &onComplete) override; - DBStatus RemoveDeviceData(const std::string &device) override; DBStatus CreateDistributedTable(const std::string &tableName) override; -- Gitee From 7729d87989908a73f1b76d80278677c157de8c59 Mon Sep 17 00:00:00 2001 From: zqq Date: Wed, 5 Jan 2022 15:34:33 +0800 Subject: [PATCH 33/53] modify sync interface in rdb test case Signed-off-by: zqq --- .../syncer/distributeddb_relational_ver_p2p_sync_test.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index 963983bbe..a9f55059b 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -150,7 +150,11 @@ namespace { std::vector devices = {DEVICE_B}; Query query = Query::Select(g_tableName); std::map> statusMap; - DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, query, statusMap); + SyncStatusCallback callBack = [&statusMap]( + const std::map> &devicesMap) { + statusMap = devicesMap; + }; + DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, query, callBack, true); EXPECT_EQ(callStatus, OK); for (const auto &tablesRes : statusMap) { for (const auto &tableStatus : tablesRes.second) { -- Gitee From 892fd7f78f8a0ebdcbc3e57689f196cff206c4c2 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Fri, 31 Dec 2021 17:44:48 +0800 Subject: [PATCH 34/53] Add UT case GetIncorrectTypeData1. Signed-off-by: lidwchn --- .../storage/src/data_transformer.cpp | 17 +- ...single_ver_relational_storage_executor.cpp | 44 +++--- ...distributeddb_relational_get_data_test.cpp | 146 ++++++++++++------ 3 files changed, 131 insertions(+), 76 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp index b004d4d60..b3e60c309 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -303,13 +303,10 @@ int DataTransformer::SerializeValue(Value &value, const RowData &rowData, const for (uint32_t i = 0; i < rowData.size(); ++i) { const auto &dataValue = rowData[i]; const auto &fieldInfo = fieldInfoList[i]; - if (dataValue.GetType() != StorageType::STORAGE_TYPE_NULL && - dataValue.GetType() != fieldInfo.GetStorageType()) { - return -E_INVALID_DATA; - } if (typeFuncMap.find(dataValue.GetType()) == typeFuncMap.end()) { return -E_NOT_SUPPORT; } + totalLength += Parcel::GetUInt32Len(); // For save the dataValue's type. uint32_t dataLength = CalDataValueLength(dataValue); totalLength += dataLength; } @@ -320,9 +317,7 @@ int DataTransformer::SerializeValue(Value &value, const RowData &rowData, const for (const auto &dataValue : rowData) { const auto &fieldInfo = fieldInfoList[index++]; StorageType type = dataValue.GetType(); - if (dataValue.GetType() == StorageType::STORAGE_TYPE_NULL) { - type = fieldInfo.GetStorageType(); - } + parcel.WriteUInt32(static_cast(type)); int errCode = typeFuncMap[type].serializeFunc(dataValue, parcel); if (errCode != E_OK) { value.clear(); @@ -347,7 +342,13 @@ int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalDa DataValue dataValue; LOGD("[DataTransformer][DeSerializeValue] start deSerialize %s type %d", fieldInfo.GetFieldName().c_str(), fieldInfo.GetStorageType()); - int errCode = typeFuncMap[fieldInfo.GetStorageType()].deSerializeFunc(dataValue, parcel); + uint32_t type = 0; + parcel.ReadUInt32(type); + auto iter = typeFuncMap.find(static_cast(type)); + if (iter == typeFuncMap.end()) { + return -E_PARSE_FAIL; + } + int errCode = iter->second.deSerializeFunc(dataValue, parcel); if (errCode != E_OK) { LOGD("[DataTransformer][DeSerializeValue] deSerialize %s failed", fieldInfo.GetFieldName().c_str()); return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index c704a5c1d..3e9f2039b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -114,51 +114,47 @@ int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(const QueryObject &qu static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) { - switch (type) { - case StorageType::STORAGE_TYPE_BOOL: { - value = static_cast(sqlite3_column_int64(statement, cid)); + int storageType = sqlite3_column_type(statement, cid); + switch (storageType) { + case SQLITE_INTEGER: { + value = static_cast(sqlite3_column_int64(statement, cid)); break; } - - case StorageType::STORAGE_TYPE_INTEGER: { - int64_t v = static_cast(sqlite3_column_int64(statement, cid)); - value = v; + case SQLITE_FLOAT: { + value = sqlite3_column_double(statement, cid); break; } - - case StorageType::STORAGE_TYPE_REAL: { - value = sqlite3_column_double(statement, cid); + case SQLITE_BLOB: { + std::vector blobValue; + (void)SQLiteUtils::GetColumnBlobValue(statement, cid, blobValue); + Blob blob; + blob.WriteBlob(blobValue.data(), static_cast(blobValue.size())); + value = blob; break; } - - case StorageType::STORAGE_TYPE_TEXT: { + case SQLITE_NULL: { + break; + } + case SQLITE3_TEXT: { const char *colValue = reinterpret_cast(sqlite3_column_text(statement, cid)); if (colValue == nullptr) { - value = std::string(); + value.ResetValue(); } else { value = std::string(colValue); } break; } - - case StorageType::STORAGE_TYPE_BLOB: { - std::vector blobValue; - (void)SQLiteUtils::GetColumnBlobValue(statement, cid, blobValue); - Blob blob; - blob.WriteBlob(blobValue.data(), static_cast(blobValue.size())); - value = blob; + default: { break; } - - default: - return; } + return; } static void BindDataValueByType(sqlite3_stmt *statement, const std::optional &value, StorageType type, int cid) { - if (value == std::nullopt) { + if (value == std::nullopt || type == StorageType::STORAGE_TYPE_NULL) { sqlite3_bind_null(statement, cid); return; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp index db75f5c58..87482189b 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -55,7 +55,8 @@ void CreateDBAndTable() return; } - const string sql { "CREATE TABLE " + g_tableName + "(key BLOB PRIMARY KEY NOT NULL, value BLOB);" }; + const string sql = + "CREATE TABLE " + g_tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; char *zErrMsg = nullptr; errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &zErrMsg); if (errCode != SQLITE_OK) { @@ -82,40 +83,20 @@ int InitRelationalStore() return dbStatus; } -int AddOrUpdateRecord(const Key &key, const Value &value) { - static const string SQL = "INSERT OR REPLACE INTO " + g_tableName + " VALUES(?,?);"; +int AddOrUpdateRecord(int64_t key, int64_t value) { sqlite3 *db = nullptr; - sqlite3_stmt *statement = nullptr; int errCode = sqlite3_open(g_storePath.c_str(), &db); - if (errCode != SQLITE_OK) { - LOGE("open db failed:%d", errCode); - errCode = SQLiteUtils::MapSQLiteErrno(errCode); - goto END; - } - errCode = SQLiteUtils::GetStatement(db, SQL, statement); - if (errCode != E_OK) { - goto END; - } - errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // 1 means key's index - if (errCode != E_OK) { - goto END; - } - errCode = SQLiteUtils::BindBlobToStatement(statement, 2, value, true); // 2 means value's index - if (errCode != E_OK) { - goto END; - } - errCode = SQLiteUtils::StepWithRetry(statement, false); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - errCode = E_OK; + if (errCode == SQLITE_OK) { + const string sql = + "INSERT OR REPLACE INTO " + g_tableName + " VALUES(" + to_string(key) + "," + to_string(value) + ");"; + errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr); } - -END: - SQLiteUtils::ResetStatement(statement, true, errCode); + errCode = SQLiteUtils::MapSQLiteErrno(errCode); sqlite3_close(db); return errCode; } -int GetLogData(const Key &key, uint64_t &flag, TimeStamp ×tamp, const DeviceID &device = "") +int GetLogData(int key, uint64_t &flag, TimeStamp ×tamp, const DeviceID &device = "") { string tableName = g_tableName; if (!device.empty()) { @@ -136,11 +117,7 @@ int GetLogData(const Key &key, uint64_t &flag, TimeStamp ×tamp, const Devic if (errCode != E_OK) { goto END; } - errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // 1 means key's index - if (errCode != E_OK) { - goto END; - } - errCode = SQLiteUtils::GetStatement(db, sql, statement); + errCode = SQLiteUtils::BindInt64ToStatement(statement, 1, key); // 1 means key's index if (errCode != E_OK) { goto END; } @@ -184,6 +161,22 @@ const RelationalSyncAbleStorage *GetRelationalStore() } return static_cast(store)->GetStorageEngine(); } + +int GetCount(sqlite3 *db, const string &sql, size_t &count) +{ + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + return errCode; + } + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + count = static_cast(sqlite3_column_int64(stmt, 0)); + errCode = E_OK; + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} } class DistributedDBRelationalGetDataTest : public testing::Test { @@ -238,15 +231,14 @@ HWTEST_F(DistributedDBRelationalGetDataTest, LogTbl1, TestSize.Level1) * @tc.steps: step1. Put data. * @tc.expected: Succeed, return OK. */ - Key insertKey = KEY_1; - Value insertValue = VALUE_1; + int insertKey = 1; + int insertValue = 1; EXPECT_EQ(AddOrUpdateRecord(insertKey, insertValue), E_OK); /** * @tc.steps: step2. Check log record. * @tc.expected: Record exists. */ - Value value; uint64_t flag = 0; TimeStamp timestamp1 = 0; EXPECT_EQ(GetLogData(insertKey, flag, timestamp1), E_OK); @@ -271,10 +263,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) */ const int RECORD_COUNT = 500; for (int i = 0; i < RECORD_COUNT; ++i) { - string key = "key_" + to_string(i); - string value = "value_" + to_string(i); - EXPECT_EQ(AddOrUpdateRecord(Key(key.data(), key.data() + key.size()), - Value(value.data(), value.data() + value.size())), E_OK); + EXPECT_EQ(AddOrUpdateRecord(i, i), E_OK); } /** @@ -317,10 +306,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) */ const int RECORD_COUNT = 100; // 100 records. for (int i = 0; i < RECORD_COUNT; ++i) { - string key = "k" + to_string(i); - string value = "v" + to_string(i); - EXPECT_EQ(AddOrUpdateRecord(Key(key.data(), key.data() + key.size()), - Value(value.data(), value.data() + value.size())), E_OK); + EXPECT_EQ(AddOrUpdateRecord(i, i), E_OK); } /** @@ -341,4 +327,76 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) EXPECT_EQ(token, nullptr); SingleVerKvEntry::Release(entries); } + +/** + * @tc.name: GetIncorrectTypeData1 + * @tc.desc: GetSyncData and PutSyncDataWithQuery interface. + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Level1) +{ + EXPECT_EQ(InitRelationalStore(), DBStatus::OK); + + /** + * @tc.steps: step1. Create distributed table "dataPlus". + * @tc.expected: Succeed, return OK. + */ + sqlite3 *db = nullptr; + EXPECT_EQ(sqlite3_open(g_storePath.c_str(), &db), SQLITE_OK); + const string tableName = g_tableName + "Plus"; + string sql = "CREATE TABLE " + tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; + ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + ASSERT_EQ(g_delegate->CreateDistributedTable(tableName, RelationalStoreDelegate::TableOption()), DBStatus::OK); + + /** + * @tc.steps: step2. Put 5 records with different type into "dataPlus" table. + * @tc.expected: Succeed, return OK. + */ + vector sqls = { + "INSERT INTO " + tableName + " VALUES(NULL, 1);", + "INSERT INTO " + tableName + " VALUES(NULL, 0.01);", + "INSERT INTO " + tableName + " VALUES(NULL, NULL);", + "INSERT INTO " + tableName + " VALUES(NULL, 'This is a text.');", + "INSERT INTO " + tableName + " VALUES(NULL, x'0123456789');", + }; + const size_t RECORD_COUNT = sqls.size(); + for (const auto &sql : sqls) { + ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + } + + /** + * @tc.steps: step3. Get all data from "dataPlus" table. + * @tc.expected: Succeed and the count is right. + */ + auto store = GetRelationalStore(); + ContinueToken token = nullptr; + QueryObject query(Query::Select(tableName)); + std::vector entries; + EXPECT_EQ(store->GetSyncData(query, SyncTimeRange {}, DataSizeSpecInfo {}, token, entries), E_OK); + EXPECT_EQ(entries.size(), RECORD_COUNT); + + /** + * @tc.steps: step4. Put data into "data" table from deviceA. + * @tc.expected: Succeed, return OK. + */ + QueryObject queryPlus(Query::Select(g_tableName)); + const DeviceID deviceID = "deviceA"; + EXPECT_EQ(const_cast(store)->PutSyncDataWithQuery(queryPlus, entries, deviceID), E_OK); + SingleVerKvEntry::Release(entries); + + /** + * @tc.steps: step5. Check data. + * @tc.expected: All data in the two tables are same. + */ + sql = "SELECT count(*) " + "FROM " + tableName + " as a, " + DBConstant::RELATIONAL_PREFIX + g_tableName + "_" + + DBCommon::TransferStringToHex(DBCommon::TransferHashString(deviceID)) + " as b " + "WHERE a.key=b.key AND (a.value=b.value OR (a.value is NULL AND b.value is NULL));"; + size_t count = 0; + EXPECT_EQ(GetCount(db, sql, count), E_OK); + EXPECT_EQ(count, RECORD_COUNT); + sqlite3_close(db); +} #endif \ No newline at end of file -- Gitee From 21303b9667e26834d69b24d5b78ade94a66a1452 Mon Sep 17 00:00:00 2001 From: lidw Date: Tue, 4 Jan 2022 06:39:10 +0000 Subject: [PATCH 35/53] Alter OpenStore interface in UT. Signed-off-by: lidwchn --- ...distributeddb_relational_get_data_test.cpp | 60 ++++++++----------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp index 87482189b..b2267d863 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -20,11 +20,11 @@ #include "db_constant.h" #include "db_errno.h" #include "db_types.h" -#include "log_print.h" #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" #include "generic_single_ver_kv_entry.h" #include "kvdb_properties.h" +#include "log_print.h" #include "relational_store_delegate.h" #include "relational_store_instance.h" #include "relational_store_manager.h" @@ -41,6 +41,7 @@ using namespace std; namespace { string g_testDir; string g_storePath; +string g_storeID = "dftStoreID"; const string g_tableName { "data" }; DistributedDB::RelationalStoreManager g_mgr(APP_ID, USER_ID); RelationalStoreDelegate *g_delegate = nullptr; @@ -56,6 +57,7 @@ void CreateDBAndTable() } const string sql = + "PRAGMA journal_mode=WAL;" "CREATE TABLE " + g_tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; char *zErrMsg = nullptr; errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &zErrMsg); @@ -66,23 +68,6 @@ void CreateDBAndTable() sqlite3_close(db); } -int InitRelationalStore() -{ - RelationalStoreDelegate *delegate = nullptr; - DBStatus dbStatus = DBStatus::DB_ERROR; - g_mgr.OpenStore(g_storePath, RelationalStoreDelegate::Option {}, - [&delegate, &dbStatus] (DBStatus status, RelationalStoreDelegate *ptr) { - delegate = ptr; - dbStatus = status; - } - ); - if (dbStatus == DBStatus::OK) { - g_delegate = delegate; - g_delegate->CreateDistributedTable(g_tableName, RelationalStoreDelegate::TableOption()); - } - return dbStatus; -} - int AddOrUpdateRecord(int64_t key, int64_t value) { sqlite3 *db = nullptr; int errCode = sqlite3_open(g_storePath.c_str(), &db); @@ -136,23 +121,22 @@ END: return errCode; } -void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, - const std::string appId, const std::string &userId, DBProperties &properties) +void InitStoreProp(const std::string &storePath, + const std::string appId, const std::string &userId, RelationalDBProperties &properties) { - properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); - properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); - properties.SetStringProp(KvDBProperties::APP_ID, appId); - properties.SetStringProp(KvDBProperties::USER_ID, userId); - properties.SetStringProp(KvDBProperties::STORE_ID, storePath); - std::string identifier = userId + "-" + appId + "-" + storePath; + properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); + properties.SetStringProp(RelationalDBProperties::APP_ID, appId); + properties.SetStringProp(RelationalDBProperties::USER_ID, userId); + properties.SetStringProp(RelationalDBProperties::STORE_ID, g_storeID); + std::string identifier = userId + "-" + appId + "-" + g_storeID; std::string hashIdentifier = DBCommon::TransferHashString(identifier); - properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); + properties.SetStringProp(RelationalDBProperties::IDENTIFIER_DATA, hashIdentifier); } const RelationalSyncAbleStorage *GetRelationalStore() { - DBProperties properties; - InitStoreProp(RelationalStoreDelegate::Option {}, g_storePath, APP_ID, USER_ID, properties); + RelationalDBProperties properties; + InitStoreProp(g_storePath, APP_ID, USER_ID, properties); int errCode = E_OK; auto store = RelationalStoreInstance::GetDataBase(properties, errCode); if (store == nullptr) { @@ -225,7 +209,9 @@ void DistributedDBRelationalGetDataTest::TearDown(void) */ HWTEST_F(DistributedDBRelationalGetDataTest, LogTbl1, TestSize.Level1) { - ASSERT_EQ(InitRelationalStore(), DBStatus::OK); + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); /** * @tc.steps: step1. Put data. @@ -255,7 +241,9 @@ HWTEST_F(DistributedDBRelationalGetDataTest, LogTbl1, TestSize.Level1) */ HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) { - ASSERT_EQ(InitRelationalStore(), DBStatus::OK); + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); /** * @tc.steps: step1. Put 500 records. @@ -298,7 +286,9 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) */ HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) { - ASSERT_EQ(InitRelationalStore(), DBStatus::OK); + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); /** * @tc.steps: step1. Put 100 records. @@ -337,7 +327,9 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) */ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Level1) { - EXPECT_EQ(InitRelationalStore(), DBStatus::OK); + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); /** * @tc.steps: step1. Create distributed table "dataPlus". @@ -348,7 +340,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Lev const string tableName = g_tableName + "Plus"; string sql = "CREATE TABLE " + tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); - ASSERT_EQ(g_delegate->CreateDistributedTable(tableName, RelationalStoreDelegate::TableOption()), DBStatus::OK); + ASSERT_EQ(g_delegate->CreateDistributedTable(tableName), DBStatus::OK); /** * @tc.steps: step2. Put 5 records with different type into "dataPlus" table. -- Gitee From 7a3335dd86e6132c54fa148e64af894ffa0a3397 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Tue, 4 Jan 2022 15:18:01 +0800 Subject: [PATCH 36/53] Change the implement of NULL value. Signed-off-by: lidwchn --- .../storage/src/data_transformer.cpp | 9 +---- ...single_ver_relational_storage_executor.cpp | 33 +++++++++---------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp index b3e60c309..f654a46f1 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -302,7 +302,6 @@ int DataTransformer::SerializeValue(Value &value, const RowData &rowData, const uint32_t totalLength = Parcel::GetUInt64Len(); // first record field count for (uint32_t i = 0; i < rowData.size(); ++i) { const auto &dataValue = rowData[i]; - const auto &fieldInfo = fieldInfoList[i]; if (typeFuncMap.find(dataValue.GetType()) == typeFuncMap.end()) { return -E_NOT_SUPPORT; } @@ -313,9 +312,7 @@ int DataTransformer::SerializeValue(Value &value, const RowData &rowData, const value.resize(totalLength); Parcel parcel(value.data(), value.size()); (void)parcel.WriteUInt64(rowData.size()); - int index = 0; for (const auto &dataValue : rowData) { - const auto &fieldInfo = fieldInfoList[index++]; StorageType type = dataValue.GetType(); parcel.WriteUInt32(static_cast(type)); int errCode = typeFuncMap[type].serializeFunc(dataValue, parcel); @@ -363,11 +360,7 @@ int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalDa if ((uint32_t)index >= valueList.size()) { return -E_INTERNAL_ERROR; // should not happen } - if (valueList[index].GetType() == StorageType::STORAGE_TYPE_NULL) { - optionalData.push_back(std::nullopt); - } else { - optionalData.push_back(valueList[index]); - } + optionalData.push_back(valueList[index]); } return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 3e9f2039b..41d337ac2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -151,54 +151,59 @@ static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, Storag return; } -static void BindDataValueByType(sqlite3_stmt *statement, - const std::optional &value, StorageType type, int cid) +static void BindDataValueByType(sqlite3_stmt *statement, const std::optional &data, int cid) { - if (value == std::nullopt || type == StorageType::STORAGE_TYPE_NULL) { - sqlite3_bind_null(statement, cid); + if (!data.has_value()) { // For the column that added after enable distributed. return; } + StorageType type = data.value().GetType(); switch (type) { case StorageType::STORAGE_TYPE_BOOL: { bool boolData = false; - value.value().GetBool(boolData); + data.value().GetBool(boolData); sqlite3_bind_int(statement, cid, boolData); break; } case StorageType::STORAGE_TYPE_INTEGER: { int64_t intData = 0; - value.value().GetInt64(intData); + data.value().GetInt64(intData); sqlite3_bind_int64(statement, cid, intData); break; } case StorageType::STORAGE_TYPE_REAL: { double doubleData = 0; - value.value().GetDouble(doubleData); + data.value().GetDouble(doubleData); sqlite3_bind_double(statement, cid, doubleData); break; } case StorageType::STORAGE_TYPE_TEXT: { std::string strData; - value.value().GetText(strData); + data.value().GetText(strData); SQLiteUtils::BindTextToStatement(statement, cid, strData); break; } case StorageType::STORAGE_TYPE_BLOB: { Blob blob; - value.value().GetBlob(blob); + data.value().GetBlob(blob); std::vector blobData(blob.GetData(), blob.GetData() + blob.GetSize()); SQLiteUtils::BindBlobToStatement(statement, cid, blobData, true); break; } + case StorageType::STORAGE_TYPE_NULL: { + sqlite3_bind_null(statement, cid); + break; + } + default: - return; + break; } + return; } static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) @@ -555,13 +560,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncDataItem(sqlite3_stmt *sta for (size_t index = 0; index < data.optionalData.size(); index++) { const auto &filedData = data.optionalData[index]; - if (filedData.has_value()) { - (void)BindDataValueByType(statement, filedData, - filedData.value().GetType(), fieldInfos[index].GetColumnId() + 1); - } else { - (void)BindDataValueByType(statement, filedData, - StorageType::STORAGE_TYPE_NULL, fieldInfos[index].GetColumnId() + 1); - } + (void)BindDataValueByType(statement, filedData, fieldInfos[index].GetColumnId() + 1); } errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); -- Gitee From fd72f96ee3ac659257fa4846318176e52aeb32fc Mon Sep 17 00:00:00 2001 From: lidwchn Date: Tue, 4 Jan 2022 19:02:11 +0800 Subject: [PATCH 37/53] Fix issue. Signed-off-by: lidwchn --- .../interfaces/include/data_value.h | 7 ++- .../interfaces/src/relational/data_value.cpp | 28 +++++++++ .../storage/src/data_transformer.cpp | 14 +++-- ...single_ver_relational_storage_executor.cpp | 62 ++++++++++++------- 4 files changed, 84 insertions(+), 27 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h index bd50513ae..50bfc8485 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h @@ -37,7 +37,10 @@ public: Blob(); ~Blob(); - DISABLE_COPY_ASSIGN_MOVE(Blob); + Blob(Blob &&); + Blob(const Blob &) = delete; + Blob &operator=(Blob &&); + Blob &operator=(const Blob &) = delete; const uint8_t* GetData() const; uint32_t GetSize() const; @@ -65,6 +68,8 @@ public: DataValue &operator=(const double &doubleVal); DataValue &operator=(const Blob &blob); DataValue &operator=(const std::string &string); + int Set(Blob *&blob); + // equals bool operator==(const DataValue &dataValue) const; bool operator!=(const DataValue &dataValue) const; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index 1aa170869..f7405885c 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -33,6 +33,21 @@ Blob::~Blob() size_ = 0; } +Blob::Blob(Blob &&blob) : ptr_(blob.ptr_), size_(blob.size_) +{ + blob.ptr_ = nullptr; + blob.size_ = 0; +} + +Blob &Blob::operator=(Blob &&blob) +{ + ptr_ = blob.ptr_; + size_ = blob.size_; + blob.ptr_ = nullptr; + blob.size_ = 0; + return *this; +} + const uint8_t *Blob::GetData() const { return ptr_; @@ -170,6 +185,19 @@ DataValue &DataValue::operator=(const Blob &blob) return *this; } +int DataValue::Set(Blob *&blob) +{ + ResetValue(); + if (blob->GetSize() <= 0) { + LOGE("Transfer Blob to DataValue failed."); + return -E_INVALID_ARGS; + } + type_ = StorageType::STORAGE_TYPE_BLOB; + value_.blobPtr = blob; + blob = nullptr; + return E_OK; +} + DataValue &DataValue::operator=(const std::string &string) { ResetValue(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp index f654a46f1..a060a8279 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -310,6 +310,9 @@ int DataTransformer::SerializeValue(Value &value, const RowData &rowData, const totalLength += dataLength; } value.resize(totalLength); + if (value.size() != totalLength) { + return -E_OUT_OF_MEMORY; + } Parcel parcel(value.data(), value.size()); (void)parcel.WriteUInt64(rowData.size()); for (const auto &dataValue : rowData) { @@ -367,10 +370,13 @@ int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalDa int DataTransformer::SerializeHashKey(Key &key, const std::string &hashKey) { - key.resize(Parcel::GetStringLen(hashKey), 0); - Parcel parcel(key.data(), Parcel::GetStringLen(hashKey)); - int errCode = parcel.WriteString(hashKey); - return errCode; + uint32_t len = Parcel::GetStringLen(hashKey); + key.resize(len, 0); + if (key.size() != len) { + return -E_OUT_OF_MEMORY; + } + Parcel parcel(key.data(), len); + return parcel.WriteString(hashKey); } int DataTransformer::DeSerializeHashKey(const Key &key, std::string &hashKey) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 41d337ac2..f702c76d1 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -112,8 +112,9 @@ int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(const QueryObject &qu return errCode; } -static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) +static int GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) { + int errCode = E_OK; int storageType = sqlite3_column_type(statement, cid); switch (storageType) { case SQLITE_INTEGER: { @@ -126,10 +127,16 @@ static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, Storag } case SQLITE_BLOB: { std::vector blobValue; - (void)SQLiteUtils::GetColumnBlobValue(statement, cid, blobValue); - Blob blob; - blob.WriteBlob(blobValue.data(), static_cast(blobValue.size())); - value = blob; + errCode = SQLiteUtils::GetColumnBlobValue(statement, cid, blobValue); + if (errCode != E_OK) { + return errCode; + } + auto blob = new (std::nothrow) Blob; + if (blob == nullptr) { + return -E_OUT_OF_MEMORY; + } + blob->WriteBlob(blobValue.data(), static_cast(blobValue.size())); + errCode = value.Set(blob); break; } case SQLITE_NULL: { @@ -141,6 +148,9 @@ static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, Storag value.ResetValue(); } else { value = std::string(colValue); + if (value.GetType() == StorageType::STORAGE_TYPE_NULL) { + errCode = -E_OUT_OF_MEMORY; + } } break; } @@ -148,62 +158,63 @@ static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, Storag break; } } - return; + return errCode; } -static void BindDataValueByType(sqlite3_stmt *statement, const std::optional &data, int cid) +static int BindDataValueByType(sqlite3_stmt *statement, const std::optional &data, int cid) { if (!data.has_value()) { // For the column that added after enable distributed. - return; + return E_OK; } + int errCode = E_OK; StorageType type = data.value().GetType(); switch (type) { case StorageType::STORAGE_TYPE_BOOL: { bool boolData = false; - data.value().GetBool(boolData); - sqlite3_bind_int(statement, cid, boolData); + (void)data.value().GetBool(boolData); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_int(statement, cid, boolData)); break; } case StorageType::STORAGE_TYPE_INTEGER: { int64_t intData = 0; - data.value().GetInt64(intData); - sqlite3_bind_int64(statement, cid, intData); + (void)data.value().GetInt64(intData); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_int64(statement, cid, intData)); break; } case StorageType::STORAGE_TYPE_REAL: { double doubleData = 0; - data.value().GetDouble(doubleData); - sqlite3_bind_double(statement, cid, doubleData); + (void)data.value().GetDouble(doubleData); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_double(statement, cid, doubleData)); break; } case StorageType::STORAGE_TYPE_TEXT: { std::string strData; - data.value().GetText(strData); - SQLiteUtils::BindTextToStatement(statement, cid, strData); + (void)data.value().GetText(strData); + errCode = SQLiteUtils::BindTextToStatement(statement, cid, strData); break; } case StorageType::STORAGE_TYPE_BLOB: { Blob blob; - data.value().GetBlob(blob); + (void)data.value().GetBlob(blob); std::vector blobData(blob.GetData(), blob.GetData() + blob.GetSize()); - SQLiteUtils::BindBlobToStatement(statement, cid, blobData, true); + errCode = SQLiteUtils::BindBlobToStatement(statement, cid, blobData, true); break; } case StorageType::STORAGE_TYPE_NULL: { - sqlite3_bind_null(statement, cid); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_null(statement, cid)); break; } default: break; } - return; + return errCode; } static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) @@ -560,7 +571,11 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncDataItem(sqlite3_stmt *sta for (size_t index = 0; index < data.optionalData.size(); index++) { const auto &filedData = data.optionalData[index]; - (void)BindDataValueByType(statement, filedData, fieldInfos[index].GetColumnId() + 1); + errCode = BindDataValueByType(statement, filedData, fieldInfos[index].GetColumnId() + 1); + if (errCode != E_OK) { + LOGE("Bind data failed, errCode:%d, cid:%d.", errCode, fieldInfos[index].GetColumnId() + 1); + return errCode; + } } errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); @@ -647,7 +662,10 @@ int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *s LOGD("[GetDataItemForSync] field:%s type:%d cid:%d", col.second.GetFieldName().c_str(), col.second.GetStorageType(), col.second.GetColumnId() + 7); DataValue value; - GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId() + 7); + errCode = GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId() + 7); + if (errCode != E_OK) { + return errCode; + } fieldInfos.push_back(col.second); data.rowData.push_back(value); } -- Gitee From bfed0223a65626b3100689a5c84b8e794b9598c8 Mon Sep 17 00:00:00 2001 From: zwtmichael Date: Thu, 6 Jan 2022 09:08:51 +0800 Subject: [PATCH 38/53] modify build problem Signed-off-by: zwtmichael --- services/distributeddataservice/libs/distributeddb/BUILD.gn | 2 +- .../distributeddataservice/libs/distributeddb/test/BUILD.gn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index 22cc35453..bd6a28f26 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -213,7 +213,7 @@ ohos_shared_library("distributeddb") { "syncer/src/multi_ver_sync_task_context.cpp", "syncer/src/multi_ver_syncer.cpp", "syncer/src/query_sync_water_mark_helper.cpp", - "syncer/src/single_ver_data_message_schedule.cpp" + "syncer/src/single_ver_data_message_schedule.cpp", "syncer/src/single_ver_data_packet.cpp", "syncer/src/single_ver_data_sync.cpp", "syncer/src/single_ver_kv_syncer.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 0f731fedb..821e536e3 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -216,7 +216,7 @@ ohos_source_set("src_file") { "../syncer/src/multi_ver_sync_task_context.cpp", "../syncer/src/multi_ver_syncer.cpp", "../syncer/src/query_sync_water_mark_helper.cpp", - "../syncer/src/single_ver_data_message_schedule.cpp" + "../syncer/src/single_ver_data_message_schedule.cpp", "../syncer/src/single_ver_data_packet.cpp", "../syncer/src/single_ver_data_sync.cpp", "../syncer/src/single_ver_kv_syncer.cpp", -- Gitee From 3c64b106e67333144d1d7efffe5e41b5692386ee Mon Sep 17 00:00:00 2001 From: lianhuix Date: Thu, 6 Jan 2022 11:11:35 +0800 Subject: [PATCH 39/53] Fix code check Signed-off-by: lianhuix --- .../relational_store_delegate_impl.cpp | 2 +- ...single_ver_relational_storage_executor.cpp | 1 - .../storage/src/sqlite/sqlite_utils.h | 12 +- .../src/single_ver_data_message_schedule.cpp | 4 +- .../src/single_ver_data_message_schedule.h | 4 +- .../syncer/src/single_ver_data_sync.cpp | 2 +- .../libs/distributeddb/test/BUILD.gn | 6 +- ...distributeddb_relational_get_data_test.cpp | 9 +- ...ributeddb_single_ver_msg_schedule_test.cpp | 14 +- .../include/distributed_rdb_tools.h | 53 +++---- .../src/distributed_rdb_tools.cpp | 111 +++++++-------- .../src/distributed_rdb_exception_test.cpp | 131 +++++++++--------- 12 files changed, 173 insertions(+), 176 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index 5ad622de2..fd81ccee0 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -149,7 +149,7 @@ void RelationalStoreDelegateImpl::OnSyncComplete(const std::mapsecond; + status = iterator->second; } table.status = status; res[device].push_back(table); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index f702c76d1..ca86ab053 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -104,7 +104,6 @@ int SQLiteSingleVerRelationalStorageExecutor::Rollback() int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(const QueryObject &query) { - int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); if (errCode != E_OK) { LOGE("[CreateDistributedTable] analysis table schema failed"); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 1c6a1ff8a..8a53c5e12 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -52,15 +52,15 @@ std::string GetTriggerModeString(TriggerModeEnum mode); } struct OpenDbProperties { - std::string uri{}; + std::string uri {}; bool createIfNecessary = true; bool isMemDb = false; - std::vector sqls{}; + std::vector sqls {}; CipherType cipherType = CipherType::AES_256_GCM; - CipherPassword passwd{}; - std::string schema{}; - std::string subdir{}; - SecurityOption securityOpt{}; + CipherPassword passwd {}; + std::string schema {}; + std::string subdir {}; + SecurityOption securityOpt {}; int conflictReslovePolicy = DEFAULT_LAST_WIN; bool createDirByStoreIdOnly = false; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp index ede29bb5a..3086ba4d3 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp @@ -112,7 +112,7 @@ void SingleVerDataMessageSchedule::ClearMsg() void SingleVerDataMessageSchedule::UpdateMsgMap() { std::queue msgTmpQueue; - { + { std::lock_guard lock(queueLock_); while (!msgQueue_.empty()) { msgTmpQueue.push(msgQueue_.front()); @@ -205,7 +205,7 @@ Message *SingleVerDataMessageSchedule::GetMsgFromMap(bool &isNeedHandle) LOGI("[DataMsgSchedule] drop msg seqId=%llu,packetId=%llu,label=%s,deviceId=%s", sequenceId, packetId, label_.c_str(), deviceId_.c_str()); delete msg; - continue; + continue; } // if packetId == finishedPacketId_ need handle // it will happened while watermark/need_abilitySync when last ack is missing diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h index e87362634..9189a7a88 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h @@ -13,6 +13,8 @@ * limitations under the License. */ +#ifndef SINGLE_VER_DATA_MESSAGE_SCHEDULE_H +#define SINGLE_VER_DATA_MESSAGE_SCHEDULE_H #include #include #include @@ -22,8 +24,6 @@ #include "runtime_context.h" #include "single_ver_sync_task_context.h" -#ifndef SINGLE_VER_DATA_MESSAGE_SCHEDULE_H -#define SINGLE_VER_DATA_MESSAGE_SCHEDULE_H namespace DistributedDB { class SingleVerDataMessageSchedule { public: diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index eca7485c8..23f669705 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -765,7 +765,7 @@ void SingleVerDataSync::UpdateSendInfo(SyncTimeRange dataTimeRange, SingleVerSyn } LOGI("[DataSync] mode=%d,start=%llu,end=%llu,deleteStart=%llu,deleteEnd=%llu,seqId=%d,packetId=%llu,window_size=%d," "isAllSend=%d,label=%s,device=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, - reSendInfo.deleteEndTime, maxSequenceIdHasSent_, reSendInfo.packetId, windowSize_,isAllDataHasSent_, + reSendInfo.deleteEndTime, maxSequenceIdHasSent_, reSendInfo.packetId, windowSize_, isAllDataHasSent_, label_.c_str(), STR_MASK(deviceId_)); } diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 821e536e3..0a46a901d 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -385,7 +385,9 @@ distributeddb_unittest("DistributedDBSingleVerP2PSyncTest") { } distributeddb_unittest("DistributedDBSingleVerMsgScheduleTest") { - sources = [ "unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp" ] + sources = [ + "unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp", + ] } distributeddb_unittest("DistributedDBInterfacesNBDelegateTest") { @@ -665,11 +667,11 @@ group("unittest") { ":DistributedDBParcelTest", ":DistributedDBSchemaObjectTest", ":DistributedDBSchemalTest", + ":DistributedDBSingleVerMsgScheduleTest", ":DistributedDBSingleVerP2PQuerySyncTest", ":DistributedDBSingleVerP2PSubscribeSyncTest", ":DistributedDBSingleVerP2PSyncCheckTest", ":DistributedDBSingleVerP2PSyncTest", - ":DistributedDBSingleVerMsgScheduleTest", ":DistributedDBSingleVersionResultSetTest", ":DistributedDBSqliteRegisterTest", ":DistributedDBStorageCommitStorageTest", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp index b2267d863..47c10ebe9 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -62,13 +62,14 @@ void CreateDBAndTable() char *zErrMsg = nullptr; errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &zErrMsg); if (errCode != SQLITE_OK) { - LOGE("sql error:%s",zErrMsg); + LOGE("sql error:%s", zErrMsg); sqlite3_free(zErrMsg); } sqlite3_close(db); } -int AddOrUpdateRecord(int64_t key, int64_t value) { +int AddOrUpdateRecord(int64_t key, int64_t value) +{ sqlite3 *db = nullptr; int errCode = sqlite3_open(g_storePath.c_str(), &db); if (errCode == SQLITE_OK) { @@ -121,8 +122,8 @@ END: return errCode; } -void InitStoreProp(const std::string &storePath, - const std::string appId, const std::string &userId, RelationalDBProperties &properties) +void InitStoreProp(const std::string &storePath, const std::string &appId, const std::string &userId, + RelationalDBProperties &properties) { properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); properties.SetStringProp(RelationalDBProperties::APP_ID, appId); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp index cff9bc90c..c5c0afa8d 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp @@ -55,7 +55,7 @@ void DistributedDBSingleVerMsgScheduleTest::TearDown(void) * @tc.name: MsgSchedule001 * @tc.desc: Test MsgSchedule funtion with normal sequenceId * @tc.type: FUNC - * @tc.require: + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule001, TestSize.Level0) @@ -105,7 +105,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule001, TestSize.Level0) * @tc.name: MsgSchedule002 * @tc.desc: Test MsgSchedule funtion with by low version * @tc.type: FUNC - * @tc.require: + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule002, TestSize.Level0) @@ -148,7 +148,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule002, TestSize.Level0) * @tc.name: MsgSchedule003 * @tc.desc: Test MsgSchedule funtion with cross sessionId * @tc.type: FUNC - * @tc.require: + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule003, TestSize.Level0) @@ -215,7 +215,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule003, TestSize.Level0) * @tc.name: MsgSchedule004 * @tc.desc: Test MsgSchedule funtion with same sessionId with different packetId * @tc.type: FUNC - * @tc.require: + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule004, TestSize.Level0) @@ -274,7 +274,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule004, TestSize.Level0) * @tc.name: MsgSchedule005 * @tc.desc: Test MsgSchedule funtion with same sessionId with different packetId * @tc.type: FUNC - * @tc.require: + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule005, TestSize.Level0) @@ -331,7 +331,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule005, TestSize.Level0) * @tc.name: MsgSchedule006 * @tc.desc: Test MsgSchedule funtion with same sessionId and same sequenceId and packetId * @tc.type: FUNC - * @tc.require: + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule006, TestSize.Level0) @@ -397,7 +397,7 @@ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule006, TestSize.Level0) * @tc.name: MsgSchedule007 * @tc.desc: Test MsgSchedule funtion with same sessionId and duplicate sequenceId and low packetId * @tc.type: FUNC - * @tc.require: + * @tc.require: * @tc.author: zhuwentao */ HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule007, TestSize.Level0) diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h index 15ce29fd9..c891afa0b 100644 --- a/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h +++ b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h @@ -38,25 +38,25 @@ struct RdbParameters { } }; -const static std::string TAG = "DistributedRdbTestTools"; - -const std::string NORMAL_PATH = "/data/test/"; -const std::string NON_EXISTENT_PATH = "/data/test/nonExistent_rdb/"; -const std::string UNREADABLE_PATH = "/data/test/unreadable_rdb/"; -const std::string UNWRITABLE_PATH = "/data/test/unwritable_rdb/"; - -const std::string NULL_STOREID = ""; -const std::string ILLEGAL_STOREID = "rdb_$%#@~%"; -const std::string MODE_STOREID = "rdb_mode"; -const std::string FULL_STOREID = "rdb_full"; -const std::string SUPER_STOREID = "rdb_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +const static std::string TAG = "DistributedRdbTools"; + +const static std::string NORMAL_PATH = "/data/test/"; +const static std::string NON_EXISTENT_PATH = "/data/test/nonExistent_rdb/"; +const static std::string UNREADABLE_PATH = "/data/test/unreadable_rdb/"; +const static std::string UNWRITABLE_PATH = "/data/test/unwritable_rdb/"; + +const static std::string NULL_STOREID = ""; +const static std::string ILLEGAL_STOREID = "rdb_$%#@~%"; +const static std::string MODE_STOREID = "rdb_mode"; +const static std::string FULL_STOREID = "rdb_full"; +const static std::string SUPER_STOREID = "rdb_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; -const std::string NON_EXISTENT_TABLE = "non_table"; -const std::string KEYWORD_START_TABLE = "naturalbase_rdb_a"; +const static std::string NON_EXISTENT_TABLE = "non_table"; +const static std::string KEYWORD_START_TABLE = "naturalbase_rdb_a"; -const std::string NORMAL_TABLE = "NORMAL_RDB"; -const std::string CONSTRAINT_TABLE = "CONSTRAINT_RDB"; +const static std::string NORMAL_TABLE = "NORMAL_RDB"; +const static std::string CONSTRAINT_TABLE = "CONSTRAINT_RDB"; const static RdbParameters g_rdbParameter1(NORMAL_PATH, DistributedDBDataGenerator::STORE_ID_1, DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); @@ -78,23 +78,28 @@ const static RdbParameters g_rdbParameter6(NON_EXISTENT_PATH, DistributedDBDataG const static RdbParameters g_rdbParameter7(UNREADABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); - + const static RdbParameters g_rdbParameter8(UNWRITABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); const static RdbParameters g_rdbParameter9(NORMAL_PATH, FULL_STOREID, DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); -class DistributedRdbTestTools final { +class DistributedRdbTools final { public: - DistributedRdbTestTools() {} - ~DistributedRdbTestTools() {} + DistributedRdbTools() {} + ~DistributedRdbTools() {} // Relational Database OpenStore And CreateDistributeTable - static DistributedDB::DBStatus GetOpenStoreStatus(const RelatetionalStoreManager *&manager, RelatetionalStoreDelegate *&delegate, - const RdbParameters ¶m); - static DistributedDB::DBStatus GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, + static DistributedDB::DBStatus GetOpenStoreStatus(const RelatetionalStoreManager *&manager, + RelatetionalStoreDelegate *&delegate, const RdbParameters ¶m); + static DistributedDB::DBStatus GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, const std::string &tableName); static bool CloseStore(const DistributedDB::RelatetionalStoreDelegate *&delegate); -private: + + static bool InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m); + static bool InitTableDataAndTrigger(const sqlite3 *&db) ; + static bool AlterTableAttributes(const sqlite3 *&db); + static bool Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name); + static void CloseSqlite3Store(sqlite3 *&db); } #endif // DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H \ No newline at end of file diff --git a/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp index 7ef56037d..fae08163e 100644 --- a/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp +++ b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp @@ -37,7 +37,8 @@ using namespace DistributedDB; using namespace DistributedDBDataGenerator; namespace { - std::string gtableNameList[33]; + const int MAX_DISTRIBUTED_TABLE_COUNT = 32; + std::string gtableNameList[MAX_DISTRIBUTED_TABLE_COUNT + 1]; const char *SQL_CREATE_NORMAL_TABLE = "CREATE TABLE IF NOT EXISTS NORMAL_RDB(" \ "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ "name VARCHAR(100) NOT NULL DEFAULT \"rdb\");"; @@ -51,11 +52,11 @@ namespace { "identity INT NOT NULL, CHECK(identity > 10000) PRIMARY KEY(id, identity)," \ "FOREGIN key(f_id) references NORMAL_RDB(id));"; - const char *SQL_CREATE_TRIGGER = "CREATE TRIGGER IF NOT EXISTS insertTrigger " \ + const char *SQL_CREATE_TRIGGER = "CREATE TRIGGER IF NOT EXISTS insertTrigger " \ "AFTER INSERT ON CONSTRAINT_RDB " \ - "FOR EACH ROW " \ - "BEGIN " \ - "update NORMAL_RDB set name = \"name_001\" " \ + "FOR EACH ROW " \ + "BEGIN " \ + "update NORMAL_RDB set name = \"name_001\" " \ "END"; const char *SQL_INSERT_NORMAL_TABLE = "INSERT INTO NORMAL_RDB (id,name)" \ @@ -68,12 +69,12 @@ namespace { const char *SQL_DROP_TABLE3 = "DROP TABLE RDB_3"; - const char *SQL_DROP_CREATE_TABLE3 = "CREATE TABLE IF NOT EXISTS RDB_3(" \ + const char *SQL_DROP_CREATE_TABLE3 = "CREATE TABLE IF NOT EXISTS RDB_3(" \ "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT);"; const char *SQL_DROP_TABLE4 = "DROP TABLE RDB_4"; - const char *SQL_DROP_CREATE_TABLE4 = "CREATE TABLE IF NOT EXISTS RDB_4(" \ + const char *SQL_DROP_CREATE_TABLE4 = "CREATE TABLE IF NOT EXISTS RDB_4(" \ "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, name CHAR(100));"; const char *SQL_JOURNAL_MODE = "PRAGMA journal_mode = DELETE;"; @@ -84,12 +85,12 @@ namespace { "VALUES (1, \'rdb_005\', \'name_rdb5\');"; } -DBStatus DistributedRdbTestTools::GetOpenStoreStatus(const RelatetionalStoreManager *&manager, RelatetionalStoreDelegate *&delegate, - const RdbParameters ¶m) +DBStatus DistributedRdbTools::GetOpenStoreStatus(const RelatetionalStoreManager *&manager, + RelatetionalStoreDelegate *&delegate, const RdbParameters ¶m) { if (manager == nullptr) { - MST_LOG("%s GetRdbStore failed! manager nullptr.", TAG.c_str()); - return DBStatus::DB_ERROR; + MST_LOG("%s GetRdbStore failed! manager nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; } if (delegate != nullptr) { delegate = nullptr; @@ -101,29 +102,29 @@ DBStatus DistributedRdbTestTools::GetOpenStoreStatus(const RelatetionalStoreMana retrun status; } -DBStatus DistributedRdbTestTools::GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, +DBStatus DistributedRdbTools::GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, const std::string &tableName) { if (delegate == nullptr) { - MST_LOG("%s CreateDistributedTable failed! delegate nullptr.", TAG.c_str()); - return DBStatus::DB_ERROR; + MST_LOG("%s CreateDistributedTable failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; } DBStatus status = delegate->CreateDistributedTable(tableName); retrun status; } -DBStatus DistributedRdbTestTools::CloseStore(const RelatetionalStoreDelegate *&delegate) +DBStatus DistributedRdbTools::CloseStore(const RelatetionalStoreDelegate *&delegate) { if (delegate == nullptr) { - MST_LOG("%s CloseStore failed! delegate nullptr.", TAG.c_str()); - return DBStatus::DB_ERROR; + MST_LOG("%s CloseStore failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; } DBStatus status = delegate->CloseStore(); return status; } -bool InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m) -{ +bool DistributedRdbTools::InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m) +{ const std::string dbName = param.path + param.storeId + ".db"; DBStributedDB::OS::RemoveFile(dbName); int errCode = sqlite3_open(dbName, &db); @@ -139,59 +140,49 @@ bool InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m) return true; } -bool InitTableDataAndTRIGGER(const sqlite3 *&db) +namespace { +void SqliteExecSql(sqlite3 *db, const char *sql) { - if (db == nullptr) { - MST_LOG("openStore Failed"); - return false; - } char *errMsg = nullptr; - int errCode1 = sqlite3_exec(db, SQL_CREATE_NORMAL_TABLE, nullptr, nullptr, &errMsg); - if (errCode1 != SQLITE_OK && errMsg != nullptr) { - MST_LOG("sqlite3_exec SQL_CREATE_NORMAL_TABLE Failed(%s)", errMsg.c_str()); - return false; - } - - int errCode2 = sqlite3_exec(db, SQL_CREATE_CONSTRAINT_TABLE, nullptr, nullptr, &errMsg); - if (errCode2 != SQLITE_OK && errMsg != nullptr) { - MST_LOG("sqlite3_exec SQL_CREATE_CONSTRAINT_TABLE Failed(%s)", errMsg.c_str()); - return false; - } - - int errCode3 = sqlite3_exec(db, SQL_CREATE_TRIGGER, nullptr, nullptr, &errMsg); - if (errCode3 != SQLITE_OK && errMsg != nullptr) { - MST_LOG("sqlite3_exec SQL_CREATE_TRIGGER Failed(%s)", errMsg.c_str()); + int errCode = sqlite3_exec(db, sql, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec sql Failed(%s)", errMsg.c_str()); return false; } + sqlite3_free(errMsg); + errMsg = nullptr; +} +} - int errCode4 = sqlite3_exec(db, SQL_INSERT_NORMAL_TABLE, nullptr, nullptr, &errMsg); - if (errCode4 != SQLITE_OK && errMsg != nullptr) { - MST_LOG("sqlite3_exec SQL_INSERT_NORMAL_TABLE Failed(%s)", errMsg.c_str()); +bool DistributedRdbTools::InitTableDataAndTrigger(const sqlite3 *&db) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); return false; } + SqliteExecSql(db, SQL_CREATE_NORMAL_TABLE); + SqliteExecSql(db, SQL_CREATE_CONSTRAINT_TABLE); + SqliteExecSql(db, SQL_CREATE_TRIGGER); + SqliteExecSql(db, SQL_INSERT_NORMAL_TABLE); - for (int i = 1; i <= 33; i++) { + for (int i = 1; i <= MAX_DISTRIBUTED_TABLE_COUNT + 1; i++) { std::string str_0 = "RDB_" + i; std::string str_1 = "CREATE TABLE IF NOT EXISTS " std::string str_2 = "( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ "name VARCHAR(100) NOT NULL, age VARCHAR(100) NOT NULL);"; - sprintf(str1,"%s%s",str0,str2); - - char *errMsg = nullptr; - int errCode1 = sqlite3_exec(db, str_1.c_str(), nullptr, nullptr, &errMsg); - if (errCode1 != SQLITE_OK && errMsg != nullptr) { - MST_LOG("sqlite3_exec PresetTables33 Failed(%s)", errMsg.c_str()); - return false; - } + std::string sql = str_1 + str_0 + str_2; + + SqliteExecSql(db, sql.c_str()); gtableNameList[i-1] = str_0; } return true; } -bool alterTableAttributes(const sqlite3 *&db) { +bool DistributedRdbTools::AlterTableAttributes(const sqlite3 *&db) +{ if (db == nullptr) { - MST_LOG("openStore Failed"); - return false; + MST_LOG("openStore Failed"); + return false; } char *errMsg = nullptr; int errCode = sqlite3_exec(db, SQL_ADD_FIELD_TABLE1, nullptr, nullptr, &errMsg); @@ -233,11 +224,11 @@ bool alterTableAttributes(const sqlite3 *&db) { } -bool Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name) -{ +bool DistributedRdbTools::Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name) +{ if (db == nullptr) { - MST_LOG("openStore Failed"); - return false; + MST_LOG("openStore Failed"); + return false; } int errCode = sqlite3_exec(db, sql_name, 0, 0, 0); if (errCode != SQLITE_OK) { @@ -247,8 +238,8 @@ bool Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name) return true; } -void CloseSqlite3Store(sqlite3 *&db) -{ +void DistributedRdbTools::CloseSqlite3Store(sqlite3 *&db) +{ if (db != nullptr) { sqlite3_close(db); db = nullptr; diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp index 3718f5378..530baf215 100644 --- a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp @@ -38,7 +38,7 @@ RelatetionalStoreDelegate *g_delegate = nullptr; RelatetionalStoreDelegate *g_delegate1 = nullptr; -class DistributedRelatinalDBExceptionOperationTest : public testing::Test { +class DistributedRDBExceptionTest : public testing::Test { public: static void SetUpTestCase(void); static void TearDownTestCase(void); @@ -47,23 +47,23 @@ public: private: }; -void DistributedRelatinalDBExceptionOperationTest::SetUpTestCase(void) +void DistributedRDBExceptionTest::SetUpTestCase(void) { - bool init1 = InitSqlite3Store(gdb, g_rdbParameter1); + bool init1 = DistributedRdbTools::InitSqlite3Store(gdb, g_rdbParameter1); ASSERT_EQ(init1, true); - bool init2 = InitTableDataAndTRIGGER(gdb); + bool init2 = DistributedRdbTools::InitTableDataAndTrigger(gdb); ASSERT_EQ(init2, true); g_manager = new (std::nothrow) RelatetionalStoreManager(g_rdbParameter1.appId, g_rdbParameter1.userId); - DBStatus status1 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate, g_rdbParameter1); + DBStatus status1 = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate, g_rdbParameter1); ASSERT_EQ(status1, DBStatus::OK); } -void DistributedRelatinalDBExceptionOperationTest::TearDownTestCase(void) +void DistributedRDBExceptionTest::TearDownTestCase(void) { CloseSqlite3Store(gdb); } -void DistributedRelatinalDBExceptionOperationTest::SetUp(void) +void DistributedRDBExceptionTest::SetUp(void) { UnitTest *test = UnitTest::GetInstance(); ASSERT_NE(test, nullptr); @@ -73,7 +73,7 @@ void DistributedRelatinalDBExceptionOperationTest::SetUp(void) MST_LOG("[SetUp] test case %s is start to run", testCaseName.c_str()); } -void DistributedRelatinalDBExceptionOperationTest::TearDown(void) +void DistributedRDBExceptionTest::TearDown(void) { MST_LOG("TearDownTestCase after case."); } @@ -85,9 +85,9 @@ void DistributedRelatinalDBExceptionOperationTest::TearDown(void) * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException001, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, dbPathException001, TestSize.Level4) { - DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter6); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter6); ASSERT_EQ(status, DBStatus::INVALID_ARGS); } @@ -98,14 +98,14 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException001, TestS * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException002, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, dbPathException002, TestSize.Level4) { sqlite3 *db = nullptr; - bool init1 = InitSqlite3Store(db, g_rdbParameter7); + bool init1 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter7); ASSERT_EQ(init1, true); bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IXUSR); ASSERT_EQ(bpermission, true); - DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter7); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter7); ASSERT_EQ(status, DBStatus::INVALID_ARGS); bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IREAD | S_IXUSR); ASSERT_EQ(bpermission, true); @@ -119,14 +119,14 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException002, TestS * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException003, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, dbPathException003, TestSize.Level4) { sqlite3 *db = nullptr; - bool init1 = InitSqlite3Store(db, g_rdbParameter8); + bool init1 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter8); ASSERT_EQ(init1, true); bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IREAD | S_IXUSR); ASSERT_EQ(bpermission, true); - DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter8); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter8); ASSERT_EQ(status, DBStatus::INVALID_ARGS); bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IWUSR | S_IREAD | S_IXUSR); ASSERT_EQ(bpermission, true); @@ -140,9 +140,9 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbPathException003, TestS * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException001, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, storeIdException001, TestSize.Level4) { - DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter2); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter2); ASSERT_EQ(status, DBStatus::INVALID_ARGS); } @@ -153,9 +153,9 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException001, Test * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException002, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, storeIdException002, TestSize.Level4) { - DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter5); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter5); ASSERT_EQ(status, DBStatus::INVALID_ARGS); } @@ -166,9 +166,9 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException002, Test * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException003, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, storeIdException003, TestSize.Level4) { - DBStatus status = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter3); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter3); ASSERT_EQ(status, DBStatus::INVALID_ARGS); } @@ -179,22 +179,22 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, storeIdException003, Test * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbModeException001, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, dbModeException001, TestSize.Level4) { - sqlite3 *db = nullptr; - bool init1 = InitSqlite3Store(db, g_rdbParameter4); + sqlite3 *db = nullptr; + bool init1 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter4); ASSERT_EQ(init1, true); - bool res1 = Sqlite3ExecOpration(db, SQL_JOURNAL_MODE); + bool res1 = DistributedRdbTools::Sqlite3ExecOpration(db, SQL_JOURNAL_MODE); ASSERT_EQ(res1, true); - DBStatus status1 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + DBStatus status1 = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); ASSERT_EQ(status1, DBStatus::NOT_SUPPORT); CloseSqlite3Store(db); - bool init2 = InitSqlite3Store(db, g_rdbParameter9); + bool init2 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter9); ASSERT_EQ(init2, true); - bool res2 = Sqlite3ExecOpration(db, SQL_SYNCHRONOUS_MODE); + bool res2 = DistributedRdbTools::Sqlite3ExecOpration(db, SQL_SYNCHRONOUS_MODE); ASSERT_EQ(res2, true); - DBStatus status2 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + DBStatus status2 = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); CloseSqlite3Store(db); } @@ -206,9 +206,9 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, dbModeException001, TestS * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException001, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, tableException001, TestSize.Level4) { - DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, NON_EXISTENT_TABLE); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, NON_EXISTENT_TABLE); ASSERT_EQ(status2, DBStatus::INVALID_ARGS); } @@ -219,9 +219,9 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException001, TestSi * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException002, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, tableException002, TestSize.Level4) { - DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, KEYWORD_START_TABLE); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, KEYWORD_START_TABLE); ASSERT_EQ(status2, DBStatus::INVALID_ARGS); } @@ -232,9 +232,9 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException002, TestSi * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException003, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, tableException003, TestSize.Level4) { - DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, NORMAL_TABLE); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, NORMAL_TABLE); ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); } @@ -245,61 +245,60 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableException003, TestSi * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, tableMulCreate001, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, tableMulCreate001, TestSize.Level4) { for (int i = 1; i <= 5; i++) { - DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[0]); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[0]); ASSERT_EQ(status2, DBStatus::OK); } } /** * @tc.name: mulTableCreate001 - * @tc.desc: dbA createDistributedTable 32 tables, successful + * @tc.desc: dbA createDistributedTable 32 tables, successful * then dbA createDistributedTable the 33'th table failed. * @tc.type: FUNC * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, mulTableCreate001, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, mulTableCreate001, TestSize.Level4) { for (auto tableName: gtableNameList) { - DBStatus status = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, tableName); + DBStatus status = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, tableName); if (tableName == gtableNameList[32]) { ASSERT_EQ(status, DBStatus::NOT_SUPPORT); } else { - ASSERT_EQ(status, DBStatus::OK); - } + ASSERT_EQ(status, DBStatus::OK); + } } } /** * @tc.name: mulTableCreate002 * @tc.desc: dbA createDistributedTable RDB_1...20 tables, And - * dbB createDistributedTable RDB_21...32 tables,successful + * dbB createDistributedTable RDB_21...32 tables,successful * then dbA createDistributedTable the 33'th table failed. * @tc.type: FUNC * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, mulTableCreate002, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, mulTableCreate002, TestSize.Level4) { RelatetionalStoreDelegate *delegateA = nullptr; - RelatetionalStoreDelegate *delegateB = nullptr; - DBStatus status1 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, delegateA, g_rdbParameter1); - ASSERT_EQ(status1, DBStatus::OK); - DBStatus status2 = DistributedRdbTestTools::GetOpenStoreStatus(g_manager, delegateB, g_rdbParameter1); - ASSERT_EQ(status2, DBStatus::OK); + RelatetionalStoreDelegate *delegateB = nullptr; + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, delegateA, g_rdbParameter1); + ASSERT_EQ(status, DBStatus::OK); + status = DistributedRdbTools::GetOpenStoreStatus(g_manager, delegateB, g_rdbParameter1); + ASSERT_EQ(status, DBStatus::OK); for (int index = 0; index < gtableNameList.size(); index++) { if (index < 20) { - DBStatus status = DistributedRdbTestTools::GetCreateDistributedTableStatus(delegateA, gtableNameList[index]); - ASSERT_EQ(status2, DBStatus::OK); - } else if (index >= 20 && index < 32) - { - DBStatus status = DistributedRdbTestTools::GetCreateDistributedTableStatus(delegateB, gtableNameList[index]); - ASSERT_EQ(status2, DBStatus::OK); + status = DistributedRdbTools::GetCreateDistributedTableStatus(delegateA, gtableNameList[index]); + ASSERT_EQ(status, DBStatus::OK); + } else if (index >= 20 && index < 32) { + status = DistributedRdbTools::GetCreateDistributedTableStatus(delegateB, gtableNameList[index]); + ASSERT_EQ(status, DBStatus::OK); } else { - ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); + ASSERT_EQ(status, DBStatus::NOT_SUPPORT); } } } @@ -311,12 +310,12 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, mulTableCreate002, TestSi * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, relatinalTable001, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, relatinalTable001, TestSize.Level4) { - bool result = alterTableAttributes(gdb); + bool result = DistributedRdbTools::AlterTableAttributes(gdb); ASSERT_EQ(result, true); for (int i = 0; i < 4; i++) { - DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[i]); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[i]); ASSERT_EQ(status2, DBStatus::OK); } } @@ -328,9 +327,9 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, relatinalTable001, TestSi * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, constraintTable001, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, constraintTable001, TestSize.Level4) { - DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, CONSTRAINT_TABLE); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, CONSTRAINT_TABLE); ASSERT_EQ(status2, DBStatus::OK); } @@ -341,13 +340,13 @@ HWTEST_F(DistributedRelatinalDBExceptionOperationTest, constraintTable001, TestS * @tc.require: SR000DORPP * @tc.author: xuhongkang */ -HWTEST_F(DistributedRelatinalDBExceptionOperationTest, constraintTable001, TestSize.Level4) +HWTEST_F(DistributedRDBExceptionTest, constraintTable001, TestSize.Level4) { - DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); ASSERT_EQ(status2, DBStatus::OK); - bool res1 = Sqlite3ExecOpration(db, SQL_INSERT_RDB5_TABLE); + bool res1 = DistributedRdbTools::Sqlite3ExecOpration(db, SQL_INSERT_RDB5_TABLE); ASSERT_EQ(res1, true); - DBStatus status2 = DistributedRdbTestTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); ASSERT_EQ(status2, DBStatus::OK); } } -- Gitee From 860b13aa27c7af5a19b7fc4988f597613c15931e Mon Sep 17 00:00:00 2001 From: zwtmichael Date: Thu, 6 Jan 2022 10:40:22 +0800 Subject: [PATCH 40/53] fix reviewBot Signed-off-by: zwtmichael --- .../distributeddb/common/src/db_ability.cpp | 6 ++-- .../src/single_ver_data_message_schedule.cpp | 35 ++++++++++--------- .../src/single_ver_data_message_schedule.h | 2 +- .../syncer/src/single_ver_data_sync.cpp | 15 +++++--- .../syncer/src/single_ver_data_sync.h | 2 ++ .../syncer/src/single_ver_kv_syncer.cpp | 2 +- 6 files changed, 36 insertions(+), 26 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp index e7c5a74ad..8ef267826 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp @@ -135,11 +135,11 @@ uint8_t DbAbility::GetAbilityItem(const AbilityItem abilityType) const dbAbility_.size()); return 0; } - int skip = 0; - // dbAbility_ bit[0..len] : low-->high + uint32_t skip = 0; + // dbAbility_ bit[0..len] : low-->high, skip range 0..7 for (uint32_t pos = iter->first; pos < (iter->first + iter->second); pos++, skip++) { if (dbAbility_[pos]) { - data += dbAbility_[pos] << skip; + data += (static_cast(dbAbility_[pos])) << skip; } } } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp index 3086ba4d3..b866028f0 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2022 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -12,11 +12,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "single_ver_data_message_schedule.h" +#include "db_common.h" #include "log_print.h" #include "version.h" #include "single_ver_data_sync.h" -#include "single_ver_data_message_schedule.h" namespace DistributedDB { SingleVerDataMessageSchedule::~SingleVerDataMessageSchedule() @@ -92,8 +93,8 @@ void SingleVerDataMessageSchedule::ScheduleInfoHandle(bool isNeedHandleStatus, b ClearMsgMapWithNoLock(); expectedSequenceId_ = 1; } else { - LOGI("[DataMsgSchedule] DealMsg seqId=%u finishedPacketId=%llu ok,label=%s,deviceId=%s{private}", - expectedSequenceId_, finishedPacketId_, label_.c_str(), deviceId_.c_str()); + LOGI("[DataMsgSchedule] DealMsg seqId=%u finishedPacketId=%llu ok,label=%s,dev=%s", expectedSequenceId_, + finishedPacketId_, label_.c_str(), STR_MASK(deviceId_)); expectedSequenceId_++; } } @@ -142,8 +143,8 @@ void SingleVerDataMessageSchedule::UpdateMsgMapInner(std::queue &msgT uint32_t sequenceId = msg->GetSequenceId(); uint64_t packetId = packet->GetPacketId(); if (prevSessionId_ != 0 && sessionId == prevSessionId_) { - LOGD("[DataMsgSchedule] recv prev sessionId msg,drop msg,label=%s,deviceId=%s{private}", - label_.c_str(), deviceId_.c_str()); + LOGD("[DataMsgSchedule] recv prev sessionId msg,drop msg,label=%s,dev=%s", label_.c_str(), + STR_MASK(deviceId_)); delete msg; continue; } @@ -157,18 +158,20 @@ void SingleVerDataMessageSchedule::UpdateMsgMapInner(std::queue &msgT } if (messageMap_.count(sequenceId) > 0) { const auto *cachePacket = messageMap_[sequenceId]->GetObject(); - LOGD("[DataMsgSchedule] msg packetId=%llu,cachePacketId=%llu,label=%s,deviceId=%s", packetId, - cachePacket->GetPacketId(), label_.c_str(), deviceId_.c_str()); - if (packetId != 0 && packetId < cachePacket->GetPacketId()) { - delete msg; - continue; + if (cachePacket != nullptr) { + LOGD("[DataMsgSchedule] msg packetId=%llu,cachePacketId=%llu,label=%s,dev=%s", packetId, + cachePacket->GetPacketId(), label_.c_str(), STR_MASK(deviceId_)); + if (packetId != 0 && packetId < cachePacket->GetPacketId()) { + delete msg; + continue; + } } delete messageMap_[sequenceId]; messageMap_[sequenceId] = nullptr; } messageMap_[sequenceId] = msg; - LOGD("[DataMsgSchedule] put into msgMap seqId=%llu,packetId=%llu,label=%s,deviceId=%s", sequenceId, - packetId, label_.c_str(), deviceId_.c_str()); + LOGD("[DataMsgSchedule] put into msgMap seqId=%llu,packetId=%llu,label=%s,dev=%s", sequenceId, packetId, + label_.c_str(), STR_MASK(deviceId_)); } } @@ -191,7 +194,7 @@ Message *SingleVerDataMessageSchedule::GetMsgFromMap(bool &isNeedHandle) if (sequenceId < expectedSequenceId_) { uint64_t revisePacketId = finishedPacketId_ - (expectedSequenceId_ - 1 - sequenceId); LOGI("[DataMsgSchedule] msg seqId=%llu less than exSeqId=%llu,pacId=%llu,revisePacId=%llu,label=%s,dev=%s", - sequenceId, expectedSequenceId_, packetId, revisePacketId, label_.c_str(), deviceId_.c_str()); + sequenceId, expectedSequenceId_, packetId, revisePacketId, label_.c_str(), STR_MASK(deviceId_)); if (packetId < revisePacketId) { delete msg; continue; @@ -202,8 +205,8 @@ Message *SingleVerDataMessageSchedule::GetMsgFromMap(bool &isNeedHandle) } if (sequenceId == expectedSequenceId_) { if (packetId < finishedPacketId_) { - LOGI("[DataMsgSchedule] drop msg seqId=%llu,packetId=%llu,label=%s,deviceId=%s", sequenceId, - packetId, label_.c_str(), deviceId_.c_str()); + LOGI("[DataMsgSchedule] drop msg seqId=%llu,packetId=%llu,label=%s,dev=%s", sequenceId, packetId, + label_.c_str(), STR_MASK(deviceId_)); delete msg; continue; } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h index 9189a7a88..bbd274eaa 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2022 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index 23f669705..05117ece3 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -830,11 +830,7 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) SyncEntry syncData; // get data errCode = GetDataWithPerformanceRecord(context, syncData); - // once get data occur E_EKEYREVOKED error, should also send request to remote dev to pull data. - if (SyncOperation::TransferSyncMode(mode) == SyncModeType::PUSH_AND_PULL && - context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && errCode == -E_EKEYREVOKED) { - errCode = E_OK; - } + TranslateErrCodeIfNeed(mode, context->GetRemoteSoftwareVersion(), errCode); if (!IsGetDataSuccessfully(errCode)) { LOGE("[DataSync][PushStart] get data failed, errCode=%d", errCode); @@ -873,6 +869,15 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) return errCode; } +void SingleVerDataSync::TranslateErrCodeIfNeed(int mode, uint32_t version, int &errCode) +{ + // once get data occur E_EKEYREVOKED error, should also send request to remote dev to pull data. + if (SyncOperation::TransferSyncMode(mode) == SyncModeType::PUSH_AND_PULL && version > SOFTWARE_VERSION_RELEASE_2_0 + && errCode == -E_EKEYREVOKED) { + errCode = E_OK; + } +} + int SingleVerDataSync::PushStart(SingleVerSyncTaskContext *context) { if (context == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h index 8f0e4b023..1a5a04bd0 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h @@ -152,6 +152,8 @@ protected: int RequestStart(SingleVerSyncTaskContext *context, int mode); + void TranslateErrCodeIfNeed(int mode, uint32_t version, int &errCode); + SyncTimeRange GetSyncDataTimeRange(SyncType syncType, SingleVerSyncTaskContext *context, const std::vector &inData, UpdateWaterMark &isUpdate); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp index 95396a116..d1c8732ac 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp @@ -160,7 +160,7 @@ void SingleVerKVSyncer::RemoteDataChanged(const std::string &device) void SingleVerKVSyncer::QueryAutoSync(const InternalSyncParma ¶m) { - LOGI("[SingleVerKVSyncer] trigger query sync_type=%u,dev=%s", param.mode, GetSyncDevicesStr(param.devices).c_str()); + LOGI("[SingleVerKVSyncer] trigger query syncmode=%u,dev=%s", param.mode, GetSyncDevicesStr(param.devices).c_str()); RefObject::IncObjRef(syncEngine_); int retCode = RuntimeContext::GetInstance()->ScheduleTask([this, param] { int errCode = Sync(param); -- Gitee From 7184dccb5a68ff46232b6e9acbf168661f370a57 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Fri, 7 Jan 2022 16:30:07 +0800 Subject: [PATCH 41/53] Ignore path when create memory db Signed-off-by: sunpeng --- .../src/kv_store_delegate_manager.cpp | 4 +++- .../sqlite_single_ver_natural_store.cpp | 3 +++ ...tributeddb_interfaces_nb_delegate_test.cpp | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp index aab890c2d..a0c0cde7c 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp @@ -86,7 +86,9 @@ namespace { properties.SetIntProp(KvDBProperties::DATABASE_TYPE, KvDBProperties::SINGLE_VER_TYPE); properties.SetBoolProp(KvDBProperties::MEMORY_MODE, option.isMemoryDb); properties.SetBoolProp(KvDBProperties::ENCRYPTED_MODE, option.isEncryptedDb); - properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); + if (!option.isMemoryDb) { // memory db ignore store path + properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); + } properties.SetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, option.createDirByStoreIdOnly); properties.SetSchema(schema); properties.SetBoolProp(KvDBProperties::CHECK_INTEGRITY, option.isNeedIntegrityCheck); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp index 80a98f372..8be0c6b31 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp @@ -372,6 +372,9 @@ int SQLiteSingleVerNaturalStore::ClearIncompleteDatabase(const KvDBProperties &k int SQLiteSingleVerNaturalStore::CheckDatabaseRecovery(const KvDBProperties &kvDBProp) { + if (kvDBProp.GetBoolProp(KvDBProperties::MEMORY_MODE, false)) { // memory status not need recovery + return E_OK; + } std::unique_ptr operation = std::make_unique(this, nullptr); (void)operation->ClearExportedTempFiles(kvDBProp); int errCode = operation->RekeyRecover(kvDBProp); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp index 65de8fe63..2fb8d300e 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp @@ -1779,4 +1779,25 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, SingleVerGetSecurityOption002, T EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); g_kvNbDelegatePtr = nullptr; EXPECT_TRUE(g_mgr.DeleteKvStore("SingleVerGetSecurityOption002") == OK); +} + +/** + * @tc.name: CreateMemoryDbWithoutPath + * @tc.desc: Create memory database without path. + * @tc.type: FUNC + * @tc.require: AR000CRAKN DTS2022010411580 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBInterfacesNBDelegateTest, CreateMemoryDbWithoutPath, TestSize.Level1) +{ + /** + * @tc.steps: step1. Create Memory database by GetKvStore without path. + * @tc.expected: step1. Create successfully. + */ + KvStoreDelegateManager mgr(APP_ID, USER_ID); + const KvStoreNbDelegate::Option option = {true, true}; + mgr.SetKvStoreConfig(g_config); + mgr.GetKvStore("memory_without_path", option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); } \ No newline at end of file -- Gitee From b87ee8f2306d1e89ba9df6db1cd097d7cd8fddcd Mon Sep 17 00:00:00 2001 From: xuhongkang Date: Thu, 6 Jan 2022 19:44:46 +0800 Subject: [PATCH 42/53] add create memory db test Signed-off-by: xuhongkang --- .../src/distributeddb_nb_create_test.cpp | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp index 538fe699f..1e196a6b8 100755 --- a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp @@ -1155,6 +1155,52 @@ HWTEST_F(DistributeddbNbCreateTest, MemoryDb002, TestSize.Level0) ReleaseManager(manager); } +/* + * @tc.name: MemoryDb 003 + * @tc.desc: verify that the memory db do not need path and can create db successfully. + * @tc.type: FUNC + * @tc.require: SR000CRAD8 + * @tc.author: wangyulong + */ +HWTEST_F(DistributeddbNbCreateTest, MemoryDb003, TestSize.Level0) +{ + /** + * @tc.steps: step1. create memory db without path. + * @tc.expected: step1. create successfully. + */ + DelegateMgrNbCallback delegateMgrCallback; + function function = bind(&DelegateMgrNbCallback::Callback, + &delegateMgrCallback, _1, _2); + KvStoreDelegateManager *manager = new (std::nothrow) KvStoreDelegateManager( + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + ASSERT_TRUE(manager != nullptr); + EXPECT_TRUE(manager->SetKvStoreConfig({.dataDir = ""})); + Option optionParam; + optionParam.isMemoryDb = true; + KvStoreNbDelegate::Option option = DistributedDBNbTestTools::TransferNbOptionType(optionParam); + manager->GetKvStore(DistributedDBDataGenerator::STORE_ID_1, option, function); + KvStoreNbDelegate* delegate = const_cast(delegateMgrCallback.GetKvStore()); + ASSERT_TRUE(delegate != nullptr); + + /** + * @tc.steps: step2. put (k1, v1) to the memory db. + * @tc.expected: step2. the memory db can be insert data successfully. + */ + EXPECT_EQ(delegate->PutLocal(KEY_1, VALUE_1), OK); + EXPECT_EQ(delegate->Put(KEY_1, VALUE_2), OK); + Value localValue, syncValue; + EXPECT_EQ(delegate->GetLocal(KEY_1, localValue), OK); + EXPECT_EQ(delegate->Get(KEY_1, syncValue), OK); + EXPECT_EQ(localValue, VALUE_1); + EXPECT_EQ(syncValue, VALUE_2); + + EXPECT_EQ(manager->CloseKvStore(delegate), OK); + delegate = nullptr; + EXPECT_EQ(manager->DeleteKvStore(STORE_ID_1), NOT_FOUND); + + ReleaseManager(manager); +} + /* * @tc.name: OptionParam 001 * @tc.desc: verify that will check the option parameter when create encrypted DB. -- Gitee From bb126d98033a12beb6d8a47928428e7f6b2277bc Mon Sep 17 00:00:00 2001 From: lidwchn Date: Fri, 7 Jan 2022 16:00:10 +0800 Subject: [PATCH 43/53] Clone indexes for distributed table. Signed-off-by: lidwchn --- ...single_ver_relational_storage_executor.cpp | 3 ++ .../storage/src/sqlite/sqlite_utils.cpp | 44 +++++++++++++++++++ .../storage/src/sqlite/sqlite_utils.h | 1 + ...distributeddb_relational_get_data_test.cpp | 31 ++++++++++--- 4 files changed, 72 insertions(+), 7 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index ca86ab053..5e22a3e31 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -436,6 +436,9 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryOb int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode == -E_NOT_FOUND) { errCode = SQLiteUtils::CreateSameStuTable(dbHandle_, object.GetTableName(), tableName, false); + if (errCode == E_OK) { + errCode = SQLiteUtils::CloneIndexes(dbHandle_, object.GetTableName(), tableName); + } } if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 88cabd62e..2f3b1a273 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -1407,6 +1407,50 @@ int SQLiteUtils::CreateSameStuTable(sqlite3 *db, const std::string &oriTableName return errCode; } +int SQLiteUtils::CloneIndexes(sqlite3 *db, const std::string &oriTableName, const std::string &newTableName) +{ + std::string sql = + "SELECT 'CREATE ' || CASE WHEN il.'unique' THEN 'UNIQUE ' ELSE '' END || 'INDEX ' || '" + + DBConstant::RELATIONAL_PREFIX + "' || il.name || ' ON ' || '" + newTableName + + "' || '(' || GROUP_CONCAT(ii.name) || ');' " + "FROM sqlite_master AS m," + "pragma_index_list(m.name) AS il," + "pragma_index_info(il.name) AS ii " + "WHERE m.type='table' AND m.name='" + oriTableName + "' AND il.origin='c' " + "GROUP BY il.name;"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Prepare the clone sql failed:%d", errCode); + return errCode; + } + + sql.clear(); + while (true) { + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + const unsigned char *indexSql = sqlite3_column_text(stmt, 0); + sql += std::string(reinterpret_cast(indexSql)); + continue; + } + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + (void)ResetStatement(stmt, true, errCode); + break; + } + + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + if (errCode != E_OK) { + LOGE("[SQLite] execute create table sql failed"); + } + return errCode; +} + int SQLiteUtils::RegisterFunction(sqlite3 *db, const std::string &funcName, int nArg, void *uData, TransactFunc &func) { if (db == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 8a53c5e12..fee4c602d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -171,6 +171,7 @@ public: static int CreateSameStuTable(sqlite3 *db, const std::string &oriTableName, const std::string &newTableName, bool isCopyData); + static int CloneIndexes(sqlite3 *db, const std::string &oriTableName, const std::string &newTableName); #endif static int DropTriggerByName(sqlite3 *db, const std::string &name); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp index 47c10ebe9..bf63aef4f 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -333,18 +333,26 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Lev ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); /** - * @tc.steps: step1. Create distributed table "dataPlus". + * @tc.steps: step1. Create 2 index for table "data". * @tc.expected: Succeed, return OK. */ sqlite3 *db = nullptr; - EXPECT_EQ(sqlite3_open(g_storePath.c_str(), &db), SQLITE_OK); + ASSERT_EQ(sqlite3_open(g_storePath.c_str(), &db), SQLITE_OK); + string sql = "CREATE INDEX index1 ON " + g_tableName + "(value);" + "CREATE UNIQUE INDEX index2 ON " + g_tableName + "(value,key);"; + ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + + /** + * @tc.steps: step2. Create distributed table "dataPlus". + * @tc.expected: Succeed, return OK. + */ const string tableName = g_tableName + "Plus"; - string sql = "CREATE TABLE " + tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; + sql = "CREATE TABLE " + tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); ASSERT_EQ(g_delegate->CreateDistributedTable(tableName), DBStatus::OK); /** - * @tc.steps: step2. Put 5 records with different type into "dataPlus" table. + * @tc.steps: step3. Put 5 records with different type into "dataPlus" table. * @tc.expected: Succeed, return OK. */ vector sqls = { @@ -360,7 +368,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Lev } /** - * @tc.steps: step3. Get all data from "dataPlus" table. + * @tc.steps: step4. Get all data from "dataPlus" table. * @tc.expected: Succeed and the count is right. */ auto store = GetRelationalStore(); @@ -371,7 +379,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Lev EXPECT_EQ(entries.size(), RECORD_COUNT); /** - * @tc.steps: step4. Put data into "data" table from deviceA. + * @tc.steps: step5. Put data into "data" table from deviceA. * @tc.expected: Succeed, return OK. */ QueryObject queryPlus(Query::Select(g_tableName)); @@ -380,7 +388,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Lev SingleVerKvEntry::Release(entries); /** - * @tc.steps: step5. Check data. + * @tc.steps: step6. Check data. * @tc.expected: All data in the two tables are same. */ sql = "SELECT count(*) " @@ -390,6 +398,15 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Lev size_t count = 0; EXPECT_EQ(GetCount(db, sql, count), E_OK); EXPECT_EQ(count, RECORD_COUNT); + + /** + * @tc.steps: step7. Check index. + * @tc.expected: 2 index for deviceA's data table exists. + */ + sql = "SELECT count(*) FROM sqlite_master WHERE type='index' AND tbl_name='" + DBConstant::RELATIONAL_PREFIX + + g_tableName + "_" + DBCommon::TransferStringToHex(DBCommon::TransferHashString(deviceID)) + "'"; + EXPECT_EQ(GetCount(db, sql, count), E_OK); + EXPECT_EQ(count, 2UL); // The index count is 2. sqlite3_close(db); } #endif \ No newline at end of file -- Gitee From 7a0f1606c32b3878502e10860a6bfbc78f7d6c23 Mon Sep 17 00:00:00 2001 From: wbq_sky Date: Thu, 6 Jan 2022 15:31:35 +0800 Subject: [PATCH 44/53] add the max log limit and the checkpoint operation interface Signed-off-by: wbq_sky --- .../common/include/db_constant.h | 4 + .../distributeddb/common/include/db_errno.h | 1 + .../distributeddb/interfaces/include/types.h | 3 + .../interfaces/src/kv_store_errno.cpp | 1 + .../src/kv_store_nb_delegate_impl.cpp | 2 + .../storage/include/kvdb_pragma.h | 2 + .../sqlite_single_ver_natural_store.cpp | 29 ++- .../sqlite/sqlite_single_ver_natural_store.h | 5 + ...te_single_ver_natural_store_connection.cpp | 58 ++++++ ...lite_single_ver_natural_store_connection.h | 5 + .../sqlite_single_ver_storage_executor.cpp | 38 +++- .../sqlite_single_ver_storage_executor.h | 4 + .../src/sqlite/sqlite_storage_engine.cpp | 1 + .../storage/src/sqlite/sqlite_utils.cpp | 10 + .../storage/src/sqlite/sqlite_utils.h | 2 + .../common/distributeddb_common_test.cpp | 20 ++ ...tributeddb_interfaces_nb_delegate_test.cpp | 174 ++++++++++++++++++ 17 files changed, 354 insertions(+), 5 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index fc0a4e6bb..cfafa5037 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -113,6 +113,10 @@ public: static constexpr int DOUBLE_PRECISION = 15; static constexpr int MAX_DISTRIBUTED_TABLE_COUNT = 32; + static constexpr uint64_t MAX_LOG_LIMIT_HIGH = 0x400000000ULL; // 16GB + static constexpr uint64_t MAX_LOG_LIMIT_LOW = 0x400000ULL; // 4MB + static constexpr uint64_t MAX_LOG_LIMIT_DEFAULT = 0x40000000ULL; // 1GB + // For relational static const std::string RELATIONAL_PREFIX; static const std::string TIMESTAMP_ALIAS; diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h index f85331478..d76ab568f 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h @@ -119,6 +119,7 @@ constexpr int E_IGNORE_DATA = (E_BASE + 95); // ignore the data changed by other constexpr int E_FORBID_CACHEDB = (E_BASE + 96); // such after rekey can not check passwd due to file control. constexpr int E_INTERCEPT_DATA_FAIL = (E_BASE + 97); // Intercept push data failed. constexpr int E_INVALID_COMPRESS_ALGO = (E_BASE + 98); // The algo is defined, but there's no implement for the algo. +constexpr int E_LOG_OVER_LIMITS = (E_BASE + 99); // The log file size is over the limits. // Num 150+ is reserved for schema related errno, since it may be added regularly constexpr int E_JSON_PARSE_FAIL = (E_BASE + 150); // Parse json fail in grammatical level constexpr int E_JSON_INSERT_PATH_EXIST = (E_BASE + 151); // Path already exist before insert diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h index dca53003d..44adec165 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h @@ -56,6 +56,7 @@ enum DBStatus { SECURITY_OPTION_CHECK_ERROR, // such as remote device's SecurityOption not equal to local SCHEMA_VIOLATE_VALUE, // Values already exist in dbFile do not match new schema INTERCEPT_DATA_FAIL, // Interceptor push data failed. + LOG_OVER_LIMITS, // Log size is over the limits. RELATIONAL_SCHEMA_NOT_FOUND, // the sync table is not a relational table RELATIONAL_SCHEMA_CHANGED, // the schema was changed }; @@ -81,6 +82,8 @@ enum PragmaCmd { RESULT_SET_CACHE_MODE, // Accept ResultSetCacheMode Type As PragmaData RESULT_SET_CACHE_MAX_SIZE, // Allowed Int Type Range [1,16], Unit MB SET_SYNC_RETRY, + SET_MAX_LOG_LIMIT, + EXEC_CHECKPOINT, }; enum ResolutionPolicyType { diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp index b827ad0ae..492ea2011 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp @@ -52,6 +52,7 @@ namespace { { -E_EKEYREVOKED, EKEYREVOKED_ERROR }, { -E_SECURITY_OPTION_CHECK_ERROR, SECURITY_OPTION_CHECK_ERROR }, { -E_INTERCEPT_DATA_FAIL, INTERCEPT_DATA_FAIL }, + { -E_LOG_OVER_LIMITS, LOG_OVER_LIMITS }, }; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp index f1f57a386..78c0e19fc 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp @@ -57,6 +57,8 @@ namespace { {RESULT_SET_CACHE_MODE, PRAGMA_RESULT_SET_CACHE_MODE}, {RESULT_SET_CACHE_MAX_SIZE, PRAGMA_RESULT_SET_CACHE_MAX_SIZE}, {SET_SYNC_RETRY, PRAGMA_SET_SYNC_RETRY}, + {SET_MAX_LOG_LIMIT, PRAGMA_SET_MAX_LOG_LIMIT}, + {EXEC_CHECKPOINT, PRAGMA_EXEC_CHECKPOINT}, }; const std::string INVALID_CONNECTION = "[KvStoreNbDelegate] Invalid connection for operation"; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h index f873b5b16..283128bda 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h @@ -49,6 +49,8 @@ enum : int { PRAGMA_ADD_EQUAL_IDENTIFIER, PRAGMA_INTERCEPT_SYNC_DATA, PRAGMA_SUBSCRIBE_QUERY, + PRAGMA_SET_MAX_LOG_LIMIT, + PRAGMA_EXEC_CHECKPOINT, }; struct PragmaSync { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp index 8be0c6b31..372e088c1 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp @@ -178,7 +178,8 @@ SQLiteSingleVerNaturalStore::SQLiteSingleVerNaturalStore() lifeTimerId_(0), autoLifeTime_(DEF_LIFE_CYCLE_TIME), createDBTime_(0), - dataInterceptor_(nullptr) + dataInterceptor_(nullptr), + maxLogLimit_(DBConstant::MAX_LOG_LIMIT_DEFAULT) {} SQLiteSingleVerNaturalStore::~SQLiteSingleVerNaturalStore() @@ -945,8 +946,21 @@ int SQLiteSingleVerNaturalStore::RemoveDeviceData(const std::string &deviceName, if (!isInSync && !CheckWritePermission()) { return -E_NOT_PERMIT; } + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetHandle(true, errCode); + if (handle == nullptr) { + LOGE("[SingleVerNStore] RemoveDeviceData get handle failed:%d", errCode); + return errCode; + } + uint64_t logFileSize = handle->GetLogFileSize(); + ReleaseHandle(handle); + if (logFileSize > GetMaxLogLimit()) { + LOGW("[SingleVerNStore] RmDevData log size[%llu] over the limit", logFileSize); + return -E_LOG_OVER_LIMITS; + } + // Call the syncer module to erase the water mark. - int errCode = EraseDeviceWaterMark(deviceName, true); + errCode = EraseDeviceWaterMark(deviceName, true); if (errCode != E_OK) { LOGE("[SingleVerNStore] erase water mark failed:%d", errCode); return errCode; @@ -2221,6 +2235,17 @@ int SQLiteSingleVerNaturalStore::RemoveSubscribe(const std::string &subscribeId) return RemoveSubscribe(std::vector {subscribeId}); } +int SQLiteSingleVerNaturalStore::SetMaxLogLimit(uint64_t limit) +{ + LOGI("Set the max log size to %llu", limit); + maxLogLimit_.store(limit); + return E_OK; +} +uint64_t SQLiteSingleVerNaturalStore::GetMaxLogLimit() const +{ + return maxLogLimit_.load(); +} + int SQLiteSingleVerNaturalStore::RemoveAllSubscribe() { int errCode = E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h index bc4b5795f..5f21bfb51 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h @@ -183,6 +183,10 @@ public: int RemoveSubscribe(const std::vector &subscribeIds) override; + int SetMaxLogLimit(uint64_t limit); + + uint64_t GetMaxLogLimit() const; + private: int CheckDatabaseRecovery(const KvDBProperties &kvDBProp); @@ -279,6 +283,7 @@ private: mutable std::shared_mutex dataInterceptorMutex_; PushDataInterceptor dataInterceptor_; + std::atomic maxLogLimit_; }; } #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp index fc54546f8..a89060f9f 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp @@ -397,6 +397,10 @@ int SQLiteSingleVerNaturalStoreConnection::Pragma(int cmd, void *parameter) return PragmaResultSetCacheMaxSize(parameter); case PRAGMA_TRIGGER_TO_MIGRATE_DATA: return PragmaTriggerToMigrateData(*static_cast(parameter)); + case PRAGMA_SET_MAX_LOG_LIMIT: + return PragmaSetMaxLogLimit(static_cast(parameter)); + case PRAGMA_EXEC_CHECKPOINT: + return ForceCheckPoint(); default: // Call Pragma() of super class. errCode = SyncAbleKvDBConnection::Pragma(cmd, parameter); @@ -724,6 +728,36 @@ int SQLiteSingleVerNaturalStoreConnection::CheckIntegrity() const return errCode; } +int SQLiteSingleVerNaturalStoreConnection::PragmaSetMaxLogLimit(uint64_t *limit) +{ + if (limit == nullptr) { + return -E_INVALID_ARGS; + } + SQLiteSingleVerNaturalStore *naturalStore = GetDB(); + if (naturalStore == nullptr) { + LOGE("[SingleVerConnection] db is nullptr for max log limit set."); + return -E_INVALID_DB; + } + if (*limit > DBConstant::MAX_LOG_LIMIT_HIGH || *limit < DBConstant::MAX_LOG_LIMIT_LOW) { + return -E_INVALID_ARGS; + } + return naturalStore->SetMaxLogLimit(*limit); +} + +int SQLiteSingleVerNaturalStoreConnection::ForceCheckPoint() const +{ + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetExecutor(true, errCode); + if (handle == nullptr) { + LOGW("Failed to get the executor for the checkpoint."); + return errCode; + } + + errCode = handle->ForceCheckPoint(); + ReleaseExecutor(handle); + return errCode; +} + int SQLiteSingleVerNaturalStoreConnection::CheckMonoStatus(OperatePerm perm) { // 1. Get the connection number @@ -1218,6 +1252,11 @@ int SQLiteSingleVerNaturalStoreConnection::StartTransactionInCacheMode() if (handle == nullptr) { return errCode; } + if (CheckLogOverLimit(handle)) { + LOGW("Over the log limit"); + ReleaseExecutor(handle); + return -E_LOG_OVER_LIMITS; + } errCode = handle->StartTransaction(TransactType::DEFERRED); if (errCode != E_OK) { ReleaseExecutor(handle); @@ -1237,6 +1276,11 @@ int SQLiteSingleVerNaturalStoreConnection::StartTransactionNormally() if (handle == nullptr) { return errCode; } + if (CheckLogOverLimit(handle)) { + LOGW("Over the log limit"); + ReleaseExecutor(handle); + return -E_LOG_OVER_LIMITS; + } if (committedData_ == nullptr) { committedData_ = new (std::nothrow) SingleVerNaturalStoreCommitNotifyData; @@ -1706,5 +1750,19 @@ int SQLiteSingleVerNaturalStoreConnection::CheckAmendValueContentForLocalProcedu return naturalStore->CheckValueAndAmendIfNeed(ValueSource::FROM_LOCAL, oriValue, amendValue, useAmendValue); } +bool SQLiteSingleVerNaturalStoreConnection::CheckLogOverLimit(SQLiteSingleVerStorageExecutor *executor) const +{ + SQLiteSingleVerNaturalStore *naturalStore = GetDB(); + if (naturalStore == nullptr || executor == nullptr) { // Not Likely + return false; + } + uint64_t logFileSize = executor->GetLogFileSize(); + bool result = logFileSize > naturalStore->GetMaxLogLimit(); + if (result) { + LOGW("Log size[%llu] over the limit", logFileSize); + } + return result; +} + DEFINE_OBJECT_TAG_FACILITIES(SQLiteSingleVerNaturalStoreConnection) } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h index 1bd8f4008..8f523352a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h @@ -196,6 +196,11 @@ private: int CheckReadDataControlled() const; bool IsFileAccessControlled() const; + int PragmaSetMaxLogLimit(uint64_t *limit); + int ForceCheckPoint() const; + + bool CheckLogOverLimit(SQLiteSingleVerStorageExecutor *executor) const; + DECLARE_OBJECT_TAG(SQLiteSingleVerNaturalStoreConnection); // ResultSet Related Info diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index 84d5b2348..da5888aa5 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -22,6 +22,7 @@ #include "db_common.h" #include "db_errno.h" #include "parcel.h" +#include "platform_specific.h" #include "runtime_context.h" #include "sqlite_single_ver_storage_executor_sql.h" @@ -1030,10 +1031,11 @@ int SQLiteSingleVerStorageExecutor::Commit() return -E_INVALID_DB; } int errCode = SQLiteUtils::CommitTransaction(dbHandle_); - if (errCode == E_OK) { - isTransactionOpen_ = false; + if (errCode != E_OK) { + return CheckCorruptedStatus(errCode); } - return CheckCorruptedStatus(errCode); + isTransactionOpen_ = false; + return E_OK; } int SQLiteSingleVerStorageExecutor::Rollback() @@ -2132,4 +2134,34 @@ int SQLiteSingleVerStorageExecutor::CheckIntegrity() const return SQLiteUtils::CheckIntegrity(dbHandle_, CHECK_DB_INTEGRITY_SQL); } + +int SQLiteSingleVerStorageExecutor::ForceCheckPoint() const +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + SQLiteUtils::ExecuteCheckPoint(dbHandle_); + return E_OK; +} + +uint64_t SQLiteSingleVerStorageExecutor::GetLogFileSize() const +{ + if (isMemDb_) { + return 0; + } + + const char *fileName = sqlite3_db_filename(dbHandle_, "main"); + if (fileName == nullptr) { + return 0; + } + std::string walName = std::string(fileName) + "-wal"; + uint64_t fileSize = 0; + int result = OS::CalFileSize(std::string(walName), fileSize); + if (result != E_OK) { + return 0; + } + LOGI("The log file size is %llu", fileSize); + return fileSize; +} + } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h index 021199aed..ee97d0b28 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h @@ -243,6 +243,10 @@ public: int GetSyncDataWithQuery(const QueryObject &query, size_t appendLength, const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange, std::vector &dataItems) const; + int ForceCheckPoint() const; + + uint64_t GetLogFileSize() const; + private: struct SaveRecordStatements { sqlite3_stmt *queryStatement = nullptr; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp index 009400728..0d6084c9a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp @@ -81,6 +81,7 @@ int SQLiteStorageEngine::CreateNewExecutor(bool isWrite, StorageExecutor *&handl dbHandle = nullptr; return errCode; } + SQLiteUtils::ExecuteCheckPoint(dbHandle); isUpdated_ = true; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 2f3b1a273..0132318c3 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -2018,6 +2018,16 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) return E_OK; } +void SQLiteUtils::ExecuteCheckPoint(sqlite3 *db) +{ + if (db == nullptr) { + return; + } + + int chkResult = sqlite3_wal_checkpoint_v2(db, nullptr, SQLITE_CHECKPOINT_TRUNCATE, nullptr, nullptr); + LOGI("SQLite checkpoint result:%d", chkResult); +} + int SQLiteUtils::CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool &isEmpty) { if (db == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index fee4c602d..ea6478b62 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -178,6 +178,8 @@ public: static int ExpandedSql(sqlite3_stmt *stmt, std::string &basicString); + static void ExecuteCheckPoint(sqlite3 *db); + static int CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool &isEmpty); private: diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp index c667a4e22..33abc9271 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp @@ -155,6 +155,26 @@ HWTEST_F(DistributedDBCommonTest, SameProcessReUnLockFile, TestSize.Level1) EXPECT_EQ(OS::FileUnlock(fd), E_OK); } +/** + * @tc.name: CalFileSizeTest + * @tc.desc: Test the file size for function test and the performance test. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBCommonTest, CalFileSizeTest, TestSize.Level1) +{ + std::string filePath = g_testDir + "/testFileSize"; + std::ofstream ofs(filePath, std::ofstream::out); + ASSERT_TRUE(ofs.good()); + ofs << "test file size"; + ofs.close(); + uint64_t fileSize = 0; + EXPECT_EQ(OS::CalFileSize(filePath, fileSize), E_OK); + EXPECT_GT(fileSize, 0ULL); + EXPECT_EQ(OS::RemoveFile(filePath), E_OK); +} + // Distributed db is not recommended to use multiple processes to access // This testcase only guard for some wrong use on current product #if defined(RUN_MULTI_PROCESS_TEST) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp index 2fb8d300e..203af429f 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp @@ -1781,6 +1781,180 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, SingleVerGetSecurityOption002, T EXPECT_TRUE(g_mgr.DeleteKvStore("SingleVerGetSecurityOption002") == OK); } +/** + * @tc.name: MaxLogLimit001 + * @tc.desc: Test the pragma cmd of the max log size limit. + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit001, TestSize.Level2) +{ + /** + * @tc.steps:step1. Create database. + * @tc.expected: step1. Returns a non-null kvstore. + */ + KvStoreNbDelegate::Option option; + g_mgr.GetKvStore("MaxLogLimit001", option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + + /** + * @tc.steps:step2. Setting the max log limit for the valid value. + * @tc.expected: step2. Returns OK. + */ + uint64_t logSize = DBConstant::MAX_LOG_LIMIT_HIGH; + PragmaData pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + + logSize = DBConstant::MAX_LOG_LIMIT_LOW; + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + + logSize = 10 * 1024 * 1024; // 10M + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + + /** + * @tc.steps:step3. Setting the max log limit for the invalid value. + * @tc.expected: step3. Returns INLIVAD_ARGS. + */ + logSize = DBConstant::MAX_LOG_LIMIT_HIGH + 1; + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), INVALID_ARGS); + + logSize = DBConstant::MAX_LOG_LIMIT_LOW - 1; + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), INVALID_ARGS); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + g_kvNbDelegatePtr = nullptr; + EXPECT_TRUE(g_mgr.DeleteKvStore("MaxLogLimit001") == OK); +} + +/** + * @tc.name: ForceCheckpoint002 + * @tc.desc: Test the checkpoint of the database. + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit002, TestSize.Level2) +{ + /** + * @tc.steps:step1. Create database. + * @tc.expected: step1. Returns a non-null kvstore. + */ + KvStoreNbDelegate::Option option; + g_mgr.GetKvStore("MaxLogLimit002", option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + + /** + * @tc.steps:step2. Put the random entry into the database. + * @tc.expected: step2. Returns OK. + */ + Key key; + Value value; + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 30); // for 30B random key + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 3 * 1024 * 1024);// 3M value + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 40); // for 40B random key + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 20); // for 20B random key + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 1 * 1024 * 1024); // 1M value + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + + /** + * @tc.steps:step3. Get the resultset. + * @tc.expected: step3. Returns OK. + */ + KvStoreResultSet *resultSet = nullptr; + EXPECT_EQ(g_kvNbDelegatePtr->GetEntries(Key{}, resultSet), OK); + EXPECT_EQ(resultSet->GetCount(), 3); // size of all the entries is 3 + EXPECT_EQ(resultSet->MoveToFirst(), true); + + /** + * @tc.steps:step4. Put more data into the database. + * @tc.expected: step4. Returns OK. + */ + uint64_t logSize = 6 * 1024 * 1024; // 6M for initial test. + PragmaData pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 10); // for 10B random key(different size) + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 3 * 1024 * 1024);// 3MB + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 15); // for 15B random key(different size) + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + + /** + * @tc.steps:step4. Put more data into the database while the log size is over the limit. + * @tc.expected: step4. Returns LOG_OVER_LIMITS. + */ + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 25); // for 25B random key(different size) + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), LOG_OVER_LIMITS); + EXPECT_EQ(g_kvNbDelegatePtr->Delete(key), LOG_OVER_LIMITS); + EXPECT_EQ(g_kvNbDelegatePtr->StartTransaction(), LOG_OVER_LIMITS); + EXPECT_EQ(g_kvNbDelegatePtr->PutLocal(key, value), LOG_OVER_LIMITS); + EXPECT_EQ(g_kvNbDelegatePtr->RemoveDeviceData("deviceA"), LOG_OVER_LIMITS); + /** + * @tc.steps:step4. Change the max log size limit, and put the data. + * @tc.expected: step4. Returns OK. + */ + logSize *= 10; // 10 multiple size + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + g_kvNbDelegatePtr->CloseResultSet(resultSet); + + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("MaxLogLimit002"), OK); +} + +/** + * @tc.name: MaxLogCheckPoint001 + * @tc.desc: Pragma the checkpoint command. + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogCheckPoint001, TestSize.Level2) +{ + /** + * @tc.steps:step1. Create database. + * @tc.expected: step1. Returns a non-null kvstore. + */ + KvStoreNbDelegate::Option option; + g_mgr.GetKvStore("MaxLogCheckPoint001", option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + + /** + * @tc.steps:step2. Put the random entry into the database. + * @tc.expected: step2. Returns OK. + */ + Key key; + Value value; + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 30); // for 30B random key(different size) + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 1 * 1024 * 1024); // 1M + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + EXPECT_EQ(g_kvNbDelegatePtr->Delete(key), OK); + + /** + * @tc.steps:step3. Get the disk file size, execute the checkpoint and get the disk file size. + * @tc.expected: step3. Returns OK and the file size is less than the size before checkpoint. + */ + uint64_t sizeBeforeChk = 0; + g_mgr.GetKvStoreDiskSize("MaxLogCheckPoint001", sizeBeforeChk); + EXPECT_GT(sizeBeforeChk, 1 * 1024 * 1024ULL); // more than 1M + int param = 0; + PragmaData paraData = static_cast(¶m); + g_kvNbDelegatePtr->Pragma(EXEC_CHECKPOINT, paraData); + uint64_t sizeAfterChk = 0; + g_mgr.GetKvStoreDiskSize("MaxLogCheckPoint001", sizeAfterChk); + EXPECT_LT(sizeAfterChk, 100 * 1024ULL); // less than 100K + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("MaxLogCheckPoint001"), OK); +} /** * @tc.name: CreateMemoryDbWithoutPath * @tc.desc: Create memory database without path. -- Gitee From 4cc0e13733b5d0bffb3b451bfbadcc3e323e8ff5 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Mon, 10 Jan 2022 11:46:46 +0800 Subject: [PATCH 45/53] fix test case logic err Signed-off-by: sunpeng --- .../interfaces/distributeddb_interfaces_nb_delegate_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp index 203af429f..3bc496983 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp @@ -1959,7 +1959,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogCheckPoint001, TestSize.Le * @tc.name: CreateMemoryDbWithoutPath * @tc.desc: Create memory database without path. * @tc.type: FUNC - * @tc.require: AR000CRAKN DTS2022010411580 + * @tc.require: AR000CRAKN * @tc.author: sunpeng */ HWTEST_F(DistributedDBInterfacesNBDelegateTest, CreateMemoryDbWithoutPath, TestSize.Level1) @@ -1970,8 +1970,8 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, CreateMemoryDbWithoutPath, TestS */ KvStoreDelegateManager mgr(APP_ID, USER_ID); const KvStoreNbDelegate::Option option = {true, true}; - mgr.SetKvStoreConfig(g_config); mgr.GetKvStore("memory_without_path", option, g_kvNbDelegateCallback); ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); EXPECT_TRUE(g_kvDelegateStatus == OK); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); } \ No newline at end of file -- Gitee From 6bb2d7619e7f4dd306f55657a09709c1a4c68158 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Wed, 12 Jan 2022 16:14:44 +0800 Subject: [PATCH 46/53] Fix some reviewbot Signed-off-by: lidwchn --- .../distributeddb/common/src/query_expression.cpp | 2 +- .../include/relational/relational_store_manager.h | 2 +- .../src/relational/relational_store_manager.cpp | 2 +- .../storage/include/relational_store_connection.h | 2 +- .../sqlite/relational/sqlite_relational_store.cpp | 4 ++-- .../distributeddb/syncer/src/single_ver_data_sync.h | 2 +- .../distributeddb_relational_get_data_test.cpp | 3 +++ .../distributeddb_storage_query_sync_test.cpp | 12 ++++++------ .../distributeddb_relational_ver_p2p_sync_test.cpp | 6 ------ .../syncer/virtual_single_ver_sync_db_Interface.cpp | 1 - 10 files changed, 16 insertions(+), 20 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp index f8be6d9c6..20e6988d6 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp @@ -51,7 +51,7 @@ void QueryExpression::AssemblyQueryInfo(const QueryObjType queryOperType, const QueryExpression::QueryExpression() : errFlag_(true), tableName_("sync_data"), // default kv type store table name - isTableNameSpecified_(false) + isTableNameSpecified_(false) // default no specify for kv type store table name {} void QueryExpression::EqualTo(const std::string& field, const QueryValueType type, const FieldValue &value) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 1fed6cbd2..e491c1e77 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -42,7 +42,7 @@ public: DB_API DBStatus DeleteStore(const std::string &path); DB_API static void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback); - + DB_API static std::string GetRelationalStoreIdentifier(const std::string &userId, const std::string &appId, const std::string &storeId); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index d2a3bd5cc..30c74099b 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -183,7 +183,7 @@ std::string RelationalStoreManager::GetRelationalStoreIdentifier(const std::stri if (!ParamCheckUtils::CheckStoreParameter(storeId, appId, userId)) { return ""; } - return DBCommon::TransferHashString(userId + "-" + appId + "-" + storeId); + return DBCommon::TransferHashString(DBCommon::GenerateIdentifierId(storeId, appId, userId)); } } // namespace DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index 7b84fc277..f3dd78fc6 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -16,13 +16,13 @@ #define RELATIONAL_STORE_CONNECTION_H #ifdef RELATIONAL_STORE -#include "relational_store_delegate.h" #include #include #include "db_types.h" #include "macro_utils.h" #include "ref_object.h" +#include "relational_store_delegate.h" namespace DistributedDB { class IRelationalStore; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index c33c73acd..2cfdb4cf7 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -108,9 +108,9 @@ int SQLiteRelationalStore::GetSchemaFromMeta() Value schemaVal; int errCode = storageEngine_->GetMetaData(schemaKey, schemaVal); if (errCode != E_OK && errCode != -E_NOT_FOUND) { - LOGE("Get relationale schema from meta table failed. %d", errCode); + LOGE("Get relational schema from meta table failed. %d", errCode); return errCode; - } else if (errCode == -E_NOT_FOUND) { + } else if (errCode == -E_NOT_FOUND || schemaVal.empty()) { LOGW("No relational schema info was found."); return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h index 1a5a04bd0..8788197e7 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h @@ -64,7 +64,7 @@ public: int Initialize(ISyncInterface *inStorage, ICommunicator *inCommunicateHandle, std::shared_ptr &inMetadata, const std::string &deviceId); - + int SyncStart(int mode, SingleVerSyncTaskContext *context); int TryContinueSync(SingleVerSyncTaskContext *context, const Message *message); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp index bf63aef4f..f0e0baa1f 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -260,6 +260,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) * @tc.expected: Succeed and the count is right. */ auto store = GetRelationalStore(); + ASSERT_EQ(store, nullptr); ContinueToken token = nullptr; QueryObject query(Query::Select(g_tableName)); std::vector entries; @@ -305,6 +306,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) * @tc.expected: Get 70 records. */ auto store = GetRelationalStore(); + ASSERT_EQ(store, nullptr); ContinueToken token = nullptr; const unsigned int LIMIT = 80; // limit as 80. const unsigned int OFFSET = 30; // offset as 30. @@ -372,6 +374,7 @@ HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Lev * @tc.expected: Succeed and the count is right. */ auto store = GetRelationalStore(); + ASSERT_EQ(store, nullptr); ContinueToken token = nullptr; QueryObject query(Query::Select(tableName)); std::vector entries; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp index 641c9fe77..5244086af 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp @@ -1127,8 +1127,8 @@ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest001, TestSize * @tc.steps:step1. Create a query object with table name is specified * @tc.expected: ok */ - Query qeury1 = Query::Select("Relatonal_table").EqualTo("field1", "abc"); - QuerySyncObject obj1(qeury1); + Query query1 = Query::Select("Relatonal_table").EqualTo("field1", "abc"); + QuerySyncObject obj1(query1); /** * @tc.steps:step2. Serialize the object @@ -1158,11 +1158,11 @@ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest001, TestSize */ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest002, TestSize.Level1) { - Query qeury1 = Query::Select("Relatonal_table1").EqualTo("field1", "abc"); - QuerySyncObject obj1(qeury1); + Query query1 = Query::Select("Relatonal_table1").EqualTo("field1", "abc"); + QuerySyncObject obj1(query1); - Query qeury2 = Query::Select("Relatonal_table2").EqualTo("field1", "abc"); - QuerySyncObject obj2(qeury2); + Query query2 = Query::Select("Relatonal_table2").EqualTo("field1", "abc"); + QuerySyncObject obj2(query2); /** * @tc.steps:step1. check object identity diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index a9f55059b..1d9bbf744 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -45,12 +45,6 @@ namespace { VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; RelationalVirtualDevice *g_deviceB = nullptr; - // the type of g_kvDelegateCallback is function - auto g_kvDelegateCallback = [](DBStatus status, RelationalStoreDelegate *delegatePtr) { - g_kvDelegateStatus = status; - g_kvDelegatePtr = delegatePtr; - }; - int GetDB(sqlite3 *&db) { int flag = SQLITE_OPEN_URI | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp index 37cc0e8c6..b3efdebe0 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp @@ -22,7 +22,6 @@ #include "generic_single_ver_kv_entry.h" #include "intercepted_data_impl.h" #include "log_print.h" -#include "meta_data.h" #include "query_object.h" #include "securec.h" -- Gitee From bdaa67df27fc1031b7ed0331f60cfcec30eecf37 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Wed, 12 Jan 2022 17:09:02 +0800 Subject: [PATCH 47/53] Fix issue. Signed-off-by: lidwchn --- .../src/kv_store_nb_delegate_impl.cpp | 17 +------------- .../interfaces/src/relational/data_value.cpp | 2 +- .../relational_store_delegate_impl.cpp | 17 +------------- .../syncer/src/sync_operation.cpp | 22 +++++++++++++++++++ .../distributeddb/syncer/src/sync_operation.h | 2 ++ 5 files changed, 27 insertions(+), 33 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp index 78c0e19fc..532740b7b 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp @@ -828,25 +828,10 @@ DBStatus KvStoreNbDelegateImpl::DeleteInner(const IOption &option, const Key &ke void KvStoreNbDelegateImpl::OnSyncComplete(const std::map &statuses, const std::function &devicesMap)> &onComplete) const { + const auto &statusMap = SyncOperation::DBStatusTransMap(); std::map result; for (const auto &pair : statuses) { DBStatus status = DB_ERROR; - static const std::map statusMap = { - { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, - { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, - { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, - { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, - { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, - { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, - { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, - { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, - { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, - { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, - { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, - { static_cast(SyncOperation::OP_INTERCEPT_DATA_FAIL), INTERCEPT_DATA_FAIL }, - { static_cast(SyncOperation::OP_MAX_LIMITS), OVER_MAX_LIMITS }, - { static_cast(SyncOperation::OP_INVALID_ARGS), INVALID_ARGS }, - }; auto iter = statusMap.find(pair.second); if (iter != statusMap.end()) { status = iter->second; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index f7405885c..616dc9ad7 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -188,7 +188,7 @@ DataValue &DataValue::operator=(const Blob &blob) int DataValue::Set(Blob *&blob) { ResetValue(); - if (blob->GetSize() <= 0) { + if (blob == nullptr || blob->GetSize() <= 0) { LOGE("Transfer Blob to DataValue failed."); return -E_INVALID_ARGS; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index fd81ccee0..17af4c70d 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -125,22 +125,7 @@ void RelationalStoreDelegateImpl::SetReleaseFlag(bool flag) void RelationalStoreDelegateImpl::OnSyncComplete(const std::map> &devicesMap, SyncStatusCallback &onComplete) { - static const std::map statusMap = { - { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, - { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, - { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, - { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, - { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, - { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, - { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, - { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, - { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, - { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, - { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, - { static_cast(SyncOperation::OP_INTERCEPT_DATA_FAIL), INTERCEPT_DATA_FAIL }, - { static_cast(SyncOperation::OP_MAX_LIMITS), OVER_MAX_LIMITS }, - { static_cast(SyncOperation::OP_INVALID_ARGS), INVALID_ARGS }, - }; + const auto &statusMap = SyncOperation::DBStatusTransMap(); std::map> res; for (const auto &[device, statusList] : devicesMap) { for (const auto &tableStatus : statusList) { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp index bdafc83ef..70477fee5 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp @@ -269,5 +269,27 @@ std::string SyncOperation::GetQueryId() const { return query_.GetIdentify(); } + + +const std::map &SyncOperation::DBStatusTransMap() +{ + static const std::map transMap = { + { static_cast(OP_FINISHED_ALL), OK }, + { static_cast(OP_TIMEOUT), TIME_OUT }, + { static_cast(OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, + { static_cast(OP_COMM_ABNORMAL), COMM_FAILURE }, + { static_cast(OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, + { static_cast(OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, + { static_cast(OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, + { static_cast(OP_BUSY_FAILURE), BUSY }, + { static_cast(OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, + { static_cast(OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, + { static_cast(OP_NOT_SUPPORT), NOT_SUPPORT }, + { static_cast(OP_INTERCEPT_DATA_FAIL), INTERCEPT_DATA_FAIL }, + { static_cast(OP_MAX_LIMITS), OVER_MAX_LIMITS }, + { static_cast(OP_INVALID_ARGS), INVALID_ARGS }, + }; + return transMap; +} DEFINE_OBJECT_TAG_FACILITIES(SyncOperation) } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h index 547d17a99..430f23f77 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h @@ -120,6 +120,8 @@ public: static SyncType GetSyncType(int mode); static int TransferSyncMode(int mode); + static const std::map &DBStatusTransMap(); + protected: virtual ~SyncOperation(); -- Gitee From 3c3e3a64531bdc6f48b4b2f3cc040eb485ddb045 Mon Sep 17 00:00:00 2001 From: zqq Date: Fri, 14 Jan 2022 15:11:45 +0800 Subject: [PATCH 48/53] fix reviewbot Signed-off-by: zqq --- .../distributeddb/interfaces/src/relational/data_value.cpp | 7 ++++++- .../libs/distributeddb/syncer/src/generic_syncer.cpp | 2 +- .../syncer/src/single_ver_relational_syncer.cpp | 4 ++-- .../syncer/src/single_ver_sync_state_machine.cpp | 3 +-- .../syncer/distributeddb_relational_ver_p2p_sync_test.cpp | 4 ++-- .../test/unittest/common/syncer/generic_virtual_device.cpp | 4 +++- .../test/unittest/common/syncer/generic_virtual_device.h | 1 - 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index 616dc9ad7..7dc7c6a45 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -399,8 +399,13 @@ int ObjectData::GetString(const std::string &fieldName, std::string &outValue) c if (errCode != E_OK) { return errCode; } + if (blob.GetSize() == 0) { + return errCode; + } outValue.resize(blob.GetSize()); - outValue.assign(blob.GetData(), blob.GetData() + blob.GetSize()); + if (blob.GetData() != nullptr) { + outValue.assign(blob.GetData(), blob.GetData() + blob.GetSize()); + } return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp index cf78835c3..dd6f1094c 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp @@ -206,7 +206,7 @@ int GenericSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) } if (!param.wait) { std::lock_guard lockGuard(syncIdLock_); - syncIdList_.push_back((int)syncId); + syncIdList_.push_back(static_cast(syncId)); } if (operation->CheckIsAllFinished()) { operation->Finished(); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp index df15a8dab..7cb09883f 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp @@ -32,7 +32,7 @@ int SingleVerRelationalSyncer::Sync(const SyncParma ¶m) int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) { - const auto &syncInterface = static_cast(syncInterface_); + const auto &syncInterface = static_cast(syncInterface_); std::vector tablesQuery; if (param.isQuerySync) { tablesQuery.push_back(param.syncQuery); @@ -101,7 +101,7 @@ void SingleVerRelationalSyncer::DoOnSubSyncComplete(const uint32_t subSyncId, co void SingleVerRelationalSyncer::DoRollBack(std::set &subSyncIdSet) { for (const auto &removeId : subSyncIdSet) { - int retCode = RemoveSyncOperation((int)removeId); + int retCode = RemoveSyncOperation(static_cast(removeId)); if (retCode != E_OK) { LOGW("[SingleVerRelationalSyncer] RemoveSyncOperation failed errCode:%d, syncId:%d", retCode, removeId); } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp index 0208d170e..a59713da0 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp @@ -615,8 +615,7 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) { TimeOffset offset = 0; - uint32_t timeout = TIME_SYNC_WAIT_TIME; - timeout = communicator_->GetTimeout(context_->GetDeviceId()); + uint32_t timeout = communicator_->GetTimeout(context_->GetDeviceId()); // If message is data sync request, we should check timeoffset. int errCode = timeSync_->GetTimeOffset(offset, timeout); if (errCode != E_OK) { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index 1d9bbf744..4ac498607 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -91,7 +91,7 @@ namespace { { sqlite3_stmt *stmt = nullptr; EXPECT_EQ(PrepareInsert(db, stmt), SQLITE_OK); - for (int i = 0; i < (int)rowData.size(); ++i) { + for (int i = 0; i < static_cast(rowData.size()); ++i) { const auto &item = rowData[i]; const int index = i + 1; int errCode; @@ -192,7 +192,7 @@ namespace { g_deviceB->GetAllSyncData(g_tableName, targetData); for (auto &item : targetData) { - for (int j = 0; j < (int)item.rowData.size(); ++j) { + for (int j = 0; j < static_cast(item.rowData.size()); ++j) { LOGD("index %d actual_val[%s] except_val[%s]", j, item.rowData[j].ToString().c_str(), rowData[j].ToString().c_str()); EXPECT_TRUE(item.rowData[j] == rowData[j]); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp index 4771e4522..012e14141 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp @@ -25,7 +25,9 @@ GenericVirtualDevice::GenericVirtualDevice(std::string deviceId) metadata_(nullptr), deviceId_(std::move(deviceId)), remoteDeviceId_("real_device"), - context_(nullptr) + context_(nullptr), + onRemoteDataChanged_(nullptr), + subManager_(nullptr) { } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h index 3605d2cfb..388a85152 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h @@ -25,7 +25,6 @@ #include "virtual_communicator_aggregator.h" namespace DistributedDB { -class VirtualCommunicatorAggregator; class GenericVirtualDevice { public: -- Gitee From d8acfe8f1e9e34663b80287758d0fc97fdd48c52 Mon Sep 17 00:00:00 2001 From: zqq Date: Sat, 15 Jan 2022 09:19:45 +0800 Subject: [PATCH 49/53] fix memory leaks in rdb test case Signed-off-by: zqq --- .../common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index 4ac498607..40584d5fa 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -107,6 +107,7 @@ namespace { EXPECT_EQ(errCode, E_OK); } EXPECT_EQ(SimulateCommitData(db, stmt), SQLITE_DONE); + sqlite3_finalize(stmt); } void GenerateValue(RowData &rowData, std::map &dataMap) -- Gitee From b98e83dd27bd54b20199932b3634c7f7c3481612 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Wed, 12 Jan 2022 15:48:05 +0800 Subject: [PATCH 50/53] Fix code reviews. Signed-off-by: lianhuix --- .../relational/relational_schema_object.h | 2 +- .../distributeddb/common/src/db_ability.cpp | 6 ++-- .../distributeddb/common/src/json_object.cpp | 2 +- .../interfaces/include/data_value.h | 2 +- .../interfaces/src/relational/data_value.cpp | 31 ++++++++++++++----- .../relational/relational_store_manager.cpp | 14 +++++---- .../src/relational_sync_able_storage.cpp | 2 +- .../relational/sqlite_relational_store.cpp | 6 ++-- .../sqlite_single_ver_storage_executor.cpp | 1 - ...tributeddb_interfaces_nb_delegate_test.cpp | 6 ++-- 10 files changed, 43 insertions(+), 29 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index ed59af074..d13b4c165 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -120,8 +120,8 @@ public: static RelationalSyncOpinion MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, const std::string &remoteSchema, uint8_t remoteSchemaType); - // The remoteOpinion.checkOnReceive is ignored + // The remoteOpinion.checkOnReceive is ignored static RelationalSyncStrategy ConcludeSyncStrategy(const RelationalSyncOpinion &localOpinion, const RelationalSyncOpinion &remoteOpinion); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp index 8ef267826..6473420fd 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp @@ -22,7 +22,7 @@ namespace DistributedDB { DbAbility::DbAbility() { - for (auto & item : ABILITYBITS) { + for (const auto &item : ABILITYBITS) { dbAbilityItemSet_.insert(item); } dbAbility_.resize(ABILITYBITS.back().first + ABILITYBITS.back().second); @@ -150,7 +150,7 @@ int DbAbility::SetAbilityItem(const AbilityItem &abilityType, uint8_t data) { auto iter = dbAbilityItemSet_.find(abilityType); if (iter != dbAbilityItemSet_.end()) { - if (data >= pow(2, iter->second)) { + if (data >= pow(2, iter->second)) { // 2: means binary LOGE("[DbAbility] value is invalid, data=%d, use_bit=%d", data, iter->second); return -E_INTERNAL_ERROR; } @@ -159,7 +159,7 @@ int DbAbility::SetAbilityItem(const AbilityItem &abilityType, uint8_t data) } int pos = iter->first; while (data) { - dbAbility_[pos] = data % 2; // means binary + dbAbility_[pos] = data % 2; // 2: means binary data = (data >> 1); pos++; } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index 5a4d0a01a..7e9b2b470 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -519,7 +519,7 @@ int JsonObject::MoveToPath(const FieldPath &inPath, Json::Value *&exact, Json::V int JsonObject::InsertField(const FieldPath &inPath, const JsonObject &inValue, bool isAppend) { - if (inPath.empty() || inPath.size() > maxNestDepth_|| !inValue.IsValid()) { + if (inPath.empty() || inPath.size() > maxNestDepth_ || !inValue.IsValid()) { return -E_INVALID_ARGS; } if (!isValid_) { diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h index 50bfc8485..ed5add667 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h @@ -39,7 +39,7 @@ public: Blob(Blob &&); Blob(const Blob &) = delete; - Blob &operator=(Blob &&); + Blob &operator=(Blob &&) noexcept; Blob &operator=(const Blob &) = delete; const uint8_t* GetData() const; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index 7dc7c6a45..96e70dd48 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -39,12 +39,15 @@ Blob::Blob(Blob &&blob) : ptr_(blob.ptr_), size_(blob.size_) blob.size_ = 0; } -Blob &Blob::operator=(Blob &&blob) +Blob &Blob::operator=(Blob &&blob) noexcept { - ptr_ = blob.ptr_; - size_ = blob.size_; - blob.ptr_ = nullptr; - blob.size_ = 0; + if (this != &blob) { + delete[] ptr_; + ptr_ = blob.ptr_; + size_ = blob.size_; + blob.ptr_ = nullptr; + blob.size_ = 0; + } return *this; } @@ -60,7 +63,13 @@ uint32_t Blob::GetSize() const int Blob::WriteBlob(const uint8_t *ptrArray, const uint32_t &size) { - if (size == 0) return E_OK; + if (ptrArray == nullptr || size == 0) { + return E_OK; + } + + delete[] ptr_; + ptr_ = nullptr; + ptr_ = new (std::nothrow) uint8_t[size]; if (ptr_ == nullptr) { return -E_OUT_OF_MEMORY; @@ -131,6 +140,10 @@ DataValue &DataValue::operator=(const DataValue &dataValue) DataValue &DataValue::operator=(DataValue &&dataValue) noexcept { + if (this == &dataValue) { + return *this; + } + ResetValue(); this->type_ = dataValue.type_; this->value_ = dataValue.value_; switch (type_) { @@ -206,7 +219,7 @@ DataValue &DataValue::operator=(const std::string &string) return *this; } type_ = StorageType::STORAGE_TYPE_TEXT; - value_.blobPtr->WriteBlob(reinterpret_cast(string.c_str()), string.size()); + value_.blobPtr->WriteBlob(reinterpret_cast(string.c_str()), string.size()); return *this; } @@ -240,7 +253,7 @@ bool DataValue::operator==(const DataValue &dataValue) const bool DataValue::operator!=(const DataValue &dataValue) const { - return !(*this==dataValue); + return !(*this == dataValue); } int DataValue::GetBool(bool &outVal) const @@ -275,6 +288,8 @@ int DataValue::GetBlob(Blob *&outVal) const if (type_ != StorageType::STORAGE_TYPE_BLOB && type_ != StorageType::STORAGE_TYPE_TEXT) { return -E_NOT_SUPPORT; } + delete outVal; + outVal = nullptr; outVal = new (std::nothrow) Blob(); if (outVal == nullptr) { return -E_OUT_OF_MEMORY; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 30c74099b..a72f5667a 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -29,10 +29,9 @@ #include "platform_specific.h" namespace DistributedDB { -RelationalStoreManager::RelationalStoreManager(const std::string &appId, const std::string &userId) - : appId_(appId), - userId_(userId) -{} +namespace { +const int GET_CONNECT_RETRY = 3; +const int RETRY_GET_CONN_INTER = 30; void InitStoreProp(const std::string &storePath, const std::string &appId, const std::string &userId, const std::string &storeId, RelationalDBProperties &properties) @@ -40,9 +39,12 @@ void InitStoreProp(const std::string &storePath, const std::string &appId, const properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); properties.SetIdentifier(userId, appId, storeId); } +} -static const int GET_CONNECT_RETRY = 3; -static const int RETRY_GET_CONN_INTER = 30; +RelationalStoreManager::RelationalStoreManager(const std::string &appId, const std::string &userId) + : appId_(appId), + userId_(userId) +{} static RelationalStoreConnection *GetOneConnectionWithRetry(const RelationalDBProperties &properties, int &errCode) { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index 2933959e9..e718b65bb 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -25,7 +25,7 @@ namespace DistributedDB { } while (0) namespace { -static constexpr float QUERY_SYNC_THRESHOLD = 0.50; +constexpr float QUERY_SYNC_THRESHOLD = 0.50; } RelationalSyncAbleStorage::RelationalSyncAbleStorage(StorageEngine *engine) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index 2cfdb4cf7..e38138f0a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -31,10 +31,8 @@ namespace { SQLiteRelationalStore::~SQLiteRelationalStore() { - if (sqliteStorageEngine_ != nullptr) { - delete sqliteStorageEngine_; - sqliteStorageEngine_ = nullptr; - } + delete sqliteStorageEngine_; + sqliteStorageEngine_ = nullptr; } // Called when a new connection created. diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index da5888aa5..dd637aaed 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -2163,5 +2163,4 @@ uint64_t SQLiteSingleVerStorageExecutor::GetLogFileSize() const LOGI("The log file size is %llu", fileSize); return fileSize; } - } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp index 3bc496983..5bf8b67f0 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp @@ -1856,7 +1856,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit002, TestSize.Level2) Key key; Value value; DistributedDBToolsUnitTest::GetRandomKeyValue(key, 30); // for 30B random key - DistributedDBToolsUnitTest::GetRandomKeyValue(value, 3 * 1024 * 1024);// 3M value + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 3 * 1024 * 1024); // 3M value EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); DistributedDBToolsUnitTest::GetRandomKeyValue(key, 40); // for 40B random key EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); @@ -1881,7 +1881,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit002, TestSize.Level2) PragmaData pragLimit = static_cast(&logSize); EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); DistributedDBToolsUnitTest::GetRandomKeyValue(key, 10); // for 10B random key(different size) - DistributedDBToolsUnitTest::GetRandomKeyValue(value, 3 * 1024 * 1024);// 3MB + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 3 * 1024 * 1024); // 3MB EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); DistributedDBToolsUnitTest::GetRandomKeyValue(key, 15); // for 15B random key(different size) EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); @@ -1935,7 +1935,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogCheckPoint001, TestSize.Le Key key; Value value; DistributedDBToolsUnitTest::GetRandomKeyValue(key, 30); // for 30B random key(different size) - DistributedDBToolsUnitTest::GetRandomKeyValue(value, 1 * 1024 * 1024); // 1M + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 1 * 1024 * 1024); // 1M EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); EXPECT_EQ(g_kvNbDelegatePtr->Delete(key), OK); -- Gitee From becc72ef6d06e70b60a9dd05872c737dff42fa78 Mon Sep 17 00:00:00 2001 From: lianhuix Date: Sat, 15 Jan 2022 15:40:17 +0800 Subject: [PATCH 51/53] Rename 'max log limit' to 'max log size' Signed-off-by: lianhuix --- .../common/include/db_constant.h | 6 ++-- .../distributeddb/interfaces/include/types.h | 2 +- .../src/kv_store_nb_delegate_impl.cpp | 2 +- .../storage/include/kvdb_pragma.h | 2 +- .../sqlite_single_ver_natural_store.cpp | 12 +++---- .../sqlite/sqlite_single_ver_natural_store.h | 6 ++-- ...te_single_ver_natural_store_connection.cpp | 12 +++---- ...lite_single_ver_natural_store_connection.h | 2 +- ...tributeddb_interfaces_nb_delegate_test.cpp | 36 +++++++++---------- 9 files changed, 40 insertions(+), 40 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index cfafa5037..75c6d6167 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -113,9 +113,9 @@ public: static constexpr int DOUBLE_PRECISION = 15; static constexpr int MAX_DISTRIBUTED_TABLE_COUNT = 32; - static constexpr uint64_t MAX_LOG_LIMIT_HIGH = 0x400000000ULL; // 16GB - static constexpr uint64_t MAX_LOG_LIMIT_LOW = 0x400000ULL; // 4MB - static constexpr uint64_t MAX_LOG_LIMIT_DEFAULT = 0x40000000ULL; // 1GB + static constexpr uint64_t MAX_LOG_SIZE_HIGH = 0x400000000ULL; // 16GB + static constexpr uint64_t MAX_LOG_SIZE_LOW = 0x400000ULL; // 4MB + static constexpr uint64_t MAX_LOG_SIZE_DEFAULT = 0x40000000ULL; // 1GB // For relational static const std::string RELATIONAL_PREFIX; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h index 44adec165..8012f771d 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h @@ -82,7 +82,7 @@ enum PragmaCmd { RESULT_SET_CACHE_MODE, // Accept ResultSetCacheMode Type As PragmaData RESULT_SET_CACHE_MAX_SIZE, // Allowed Int Type Range [1,16], Unit MB SET_SYNC_RETRY, - SET_MAX_LOG_LIMIT, + SET_MAX_LOG_SIZE, EXEC_CHECKPOINT, }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp index 532740b7b..9c346ccdc 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp @@ -57,7 +57,7 @@ namespace { {RESULT_SET_CACHE_MODE, PRAGMA_RESULT_SET_CACHE_MODE}, {RESULT_SET_CACHE_MAX_SIZE, PRAGMA_RESULT_SET_CACHE_MAX_SIZE}, {SET_SYNC_RETRY, PRAGMA_SET_SYNC_RETRY}, - {SET_MAX_LOG_LIMIT, PRAGMA_SET_MAX_LOG_LIMIT}, + {SET_MAX_LOG_SIZE, PRAGMA_SET_MAX_LOG_SIZE}, {EXEC_CHECKPOINT, PRAGMA_EXEC_CHECKPOINT}, }; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h index 283128bda..5759240ef 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h @@ -49,7 +49,7 @@ enum : int { PRAGMA_ADD_EQUAL_IDENTIFIER, PRAGMA_INTERCEPT_SYNC_DATA, PRAGMA_SUBSCRIBE_QUERY, - PRAGMA_SET_MAX_LOG_LIMIT, + PRAGMA_SET_MAX_LOG_SIZE, PRAGMA_EXEC_CHECKPOINT, }; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp index 372e088c1..e0796c3d8 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp @@ -179,7 +179,7 @@ SQLiteSingleVerNaturalStore::SQLiteSingleVerNaturalStore() autoLifeTime_(DEF_LIFE_CYCLE_TIME), createDBTime_(0), dataInterceptor_(nullptr), - maxLogLimit_(DBConstant::MAX_LOG_LIMIT_DEFAULT) + maxLogSize_(DBConstant::MAX_LOG_SIZE_DEFAULT) {} SQLiteSingleVerNaturalStore::~SQLiteSingleVerNaturalStore() @@ -954,7 +954,7 @@ int SQLiteSingleVerNaturalStore::RemoveDeviceData(const std::string &deviceName, } uint64_t logFileSize = handle->GetLogFileSize(); ReleaseHandle(handle); - if (logFileSize > GetMaxLogLimit()) { + if (logFileSize > GetMaxLogSize()) { LOGW("[SingleVerNStore] RmDevData log size[%llu] over the limit", logFileSize); return -E_LOG_OVER_LIMITS; } @@ -2235,15 +2235,15 @@ int SQLiteSingleVerNaturalStore::RemoveSubscribe(const std::string &subscribeId) return RemoveSubscribe(std::vector {subscribeId}); } -int SQLiteSingleVerNaturalStore::SetMaxLogLimit(uint64_t limit) +int SQLiteSingleVerNaturalStore::SetMaxLogSize(uint64_t limit) { LOGI("Set the max log size to %llu", limit); - maxLogLimit_.store(limit); + maxLogSize_.store(limit); return E_OK; } -uint64_t SQLiteSingleVerNaturalStore::GetMaxLogLimit() const +uint64_t SQLiteSingleVerNaturalStore::GetMaxLogSize() const { - return maxLogLimit_.load(); + return maxLogSize_.load(); } int SQLiteSingleVerNaturalStore::RemoveAllSubscribe() diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h index 5f21bfb51..8e02212b2 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h @@ -183,9 +183,9 @@ public: int RemoveSubscribe(const std::vector &subscribeIds) override; - int SetMaxLogLimit(uint64_t limit); + int SetMaxLogSize(uint64_t limit); - uint64_t GetMaxLogLimit() const; + uint64_t GetMaxLogSize() const; private: int CheckDatabaseRecovery(const KvDBProperties &kvDBProp); @@ -283,7 +283,7 @@ private: mutable std::shared_mutex dataInterceptorMutex_; PushDataInterceptor dataInterceptor_; - std::atomic maxLogLimit_; + std::atomic maxLogSize_; }; } #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp index a89060f9f..40a692b64 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp @@ -397,8 +397,8 @@ int SQLiteSingleVerNaturalStoreConnection::Pragma(int cmd, void *parameter) return PragmaResultSetCacheMaxSize(parameter); case PRAGMA_TRIGGER_TO_MIGRATE_DATA: return PragmaTriggerToMigrateData(*static_cast(parameter)); - case PRAGMA_SET_MAX_LOG_LIMIT: - return PragmaSetMaxLogLimit(static_cast(parameter)); + case PRAGMA_SET_MAX_LOG_SIZE: + return PragmaSetMaxLogSize(static_cast(parameter)); case PRAGMA_EXEC_CHECKPOINT: return ForceCheckPoint(); default: @@ -728,7 +728,7 @@ int SQLiteSingleVerNaturalStoreConnection::CheckIntegrity() const return errCode; } -int SQLiteSingleVerNaturalStoreConnection::PragmaSetMaxLogLimit(uint64_t *limit) +int SQLiteSingleVerNaturalStoreConnection::PragmaSetMaxLogSize(uint64_t *limit) { if (limit == nullptr) { return -E_INVALID_ARGS; @@ -738,10 +738,10 @@ int SQLiteSingleVerNaturalStoreConnection::PragmaSetMaxLogLimit(uint64_t *limit) LOGE("[SingleVerConnection] db is nullptr for max log limit set."); return -E_INVALID_DB; } - if (*limit > DBConstant::MAX_LOG_LIMIT_HIGH || *limit < DBConstant::MAX_LOG_LIMIT_LOW) { + if (*limit > DBConstant::MAX_LOG_SIZE_HIGH || *limit < DBConstant::MAX_LOG_SIZE_LOW) { return -E_INVALID_ARGS; } - return naturalStore->SetMaxLogLimit(*limit); + return naturalStore->SetMaxLogSize(*limit); } int SQLiteSingleVerNaturalStoreConnection::ForceCheckPoint() const @@ -1757,7 +1757,7 @@ bool SQLiteSingleVerNaturalStoreConnection::CheckLogOverLimit(SQLiteSingleVerSto return false; } uint64_t logFileSize = executor->GetLogFileSize(); - bool result = logFileSize > naturalStore->GetMaxLogLimit(); + bool result = logFileSize > naturalStore->GetMaxLogSize(); if (result) { LOGW("Log size[%llu] over the limit", logFileSize); } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h index 8f523352a..82a5fb6e4 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h @@ -196,7 +196,7 @@ private: int CheckReadDataControlled() const; bool IsFileAccessControlled() const; - int PragmaSetMaxLogLimit(uint64_t *limit); + int PragmaSetMaxLogSize(uint64_t *limit); int ForceCheckPoint() const; bool CheckLogOverLimit(SQLiteSingleVerStorageExecutor *executor) const; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp index 5bf8b67f0..2080a5e4f 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp @@ -1782,20 +1782,20 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, SingleVerGetSecurityOption002, T } /** - * @tc.name: MaxLogLimit001 + * @tc.name: MaxLogSize001 * @tc.desc: Test the pragma cmd of the max log size limit. * @tc.type: FUNC * @tc.require: * @tc.author: wangbingquan */ -HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit001, TestSize.Level2) +HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogSize001, TestSize.Level2) { /** * @tc.steps:step1. Create database. * @tc.expected: step1. Returns a non-null kvstore. */ KvStoreNbDelegate::Option option; - g_mgr.GetKvStore("MaxLogLimit001", option, g_kvNbDelegateCallback); + g_mgr.GetKvStore("MaxLogSize001", option, g_kvNbDelegateCallback); ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); EXPECT_TRUE(g_kvDelegateStatus == OK); @@ -1803,32 +1803,32 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit001, TestSize.Level2) * @tc.steps:step2. Setting the max log limit for the valid value. * @tc.expected: step2. Returns OK. */ - uint64_t logSize = DBConstant::MAX_LOG_LIMIT_HIGH; + uint64_t logSize = DBConstant::MAX_LOG_SIZE_HIGH; PragmaData pragLimit = static_cast(&logSize); - EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); - logSize = DBConstant::MAX_LOG_LIMIT_LOW; + logSize = DBConstant::MAX_LOG_SIZE_LOW; pragLimit = static_cast(&logSize); - EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); logSize = 10 * 1024 * 1024; // 10M pragLimit = static_cast(&logSize); - EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); /** * @tc.steps:step3. Setting the max log limit for the invalid value. * @tc.expected: step3. Returns INLIVAD_ARGS. */ - logSize = DBConstant::MAX_LOG_LIMIT_HIGH + 1; + logSize = DBConstant::MAX_LOG_SIZE_HIGH + 1; pragLimit = static_cast(&logSize); - EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), INVALID_ARGS); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), INVALID_ARGS); - logSize = DBConstant::MAX_LOG_LIMIT_LOW - 1; + logSize = DBConstant::MAX_LOG_SIZE_LOW - 1; pragLimit = static_cast(&logSize); - EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), INVALID_ARGS); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), INVALID_ARGS); EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); g_kvNbDelegatePtr = nullptr; - EXPECT_TRUE(g_mgr.DeleteKvStore("MaxLogLimit001") == OK); + EXPECT_TRUE(g_mgr.DeleteKvStore("MaxLogSize001") == OK); } /** @@ -1838,14 +1838,14 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit001, TestSize.Level2) * @tc.require: * @tc.author: wangbingquan */ -HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit002, TestSize.Level2) +HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogSize002, TestSize.Level2) { /** * @tc.steps:step1. Create database. * @tc.expected: step1. Returns a non-null kvstore. */ KvStoreNbDelegate::Option option; - g_mgr.GetKvStore("MaxLogLimit002", option, g_kvNbDelegateCallback); + g_mgr.GetKvStore("MaxLogSize002", option, g_kvNbDelegateCallback); ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); EXPECT_TRUE(g_kvDelegateStatus == OK); @@ -1879,7 +1879,7 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit002, TestSize.Level2) */ uint64_t logSize = 6 * 1024 * 1024; // 6M for initial test. PragmaData pragLimit = static_cast(&logSize); - EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); DistributedDBToolsUnitTest::GetRandomKeyValue(key, 10); // for 10B random key(different size) DistributedDBToolsUnitTest::GetRandomKeyValue(value, 3 * 1024 * 1024); // 3MB EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); @@ -1902,12 +1902,12 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogLimit002, TestSize.Level2) */ logSize *= 10; // 10 multiple size pragLimit = static_cast(&logSize); - EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_LIMIT, pragLimit), OK); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); g_kvNbDelegatePtr->CloseResultSet(resultSet); EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); - EXPECT_EQ(g_mgr.DeleteKvStore("MaxLogLimit002"), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("MaxLogSize002"), OK); } /** -- Gitee From b33ab76fb32f012869761d2cd439cbbc49ebdd6f Mon Sep 17 00:00:00 2001 From: lianhuix Date: Mon, 17 Jan 2022 10:17:43 +0800 Subject: [PATCH 52/53] Fix code reviews Signed-off-by: lianhuix --- .../src/relational/relational_store_delegate_impl.cpp | 6 +++--- .../src/relational/relational_store_delegate_impl.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index 17af4c70d..a4d279cdf 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -122,13 +122,13 @@ void RelationalStoreDelegateImpl::SetReleaseFlag(bool flag) releaseFlag_ = flag; } -void RelationalStoreDelegateImpl::OnSyncComplete(const std::map> &devicesMap, +void RelationalStoreDelegateImpl::OnSyncComplete(const std::map> &devicesStatus, SyncStatusCallback &onComplete) { const auto &statusMap = SyncOperation::DBStatusTransMap(); std::map> res; - for (const auto &[device, statusList] : devicesMap) { - for (const auto &tableStatus : statusList) { + for (const auto &[device, tablesStatus] : devicesStatus) { + for (const auto &tableStatus : tablesStatus) { TableStatus table; table.tableName = tableStatus.tableName; DBStatus status = DB_ERROR; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 19911f74b..76eb8defe 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -48,7 +48,7 @@ public: void SetReleaseFlag(bool flag); private: - static void OnSyncComplete(const std::map> &devicesMap, + static void OnSyncComplete(const std::map> &devicesStatus, SyncStatusCallback &onComplete); RelationalStoreConnection *conn_ = nullptr; -- Gitee From c805469722f6e047fee73834d3047c29559d76f9 Mon Sep 17 00:00:00 2001 From: lidwchn Date: Fri, 14 Jan 2022 16:54:58 +0800 Subject: [PATCH 53/53] Fix reviewbot. Signed-off-by: lidwchn --- .../common/include/db_constant.h | 5 +++ .../storage/src/data_transformer.cpp | 8 ++-- .../src/relational_sync_able_storage.cpp | 6 +-- .../relational/sqlite_relational_store.cpp | 2 +- .../relational/sqlite_relational_store.h | 4 +- .../sqlite_single_relational_storage_engine.h | 6 --- .../sqlite_single_ver_natural_store.cpp | 6 +-- ...e_single_ver_relational_continue_token.cpp | 3 +- ...single_ver_relational_storage_executor.cpp | 43 +++++++++---------- .../storage/src/sqlite/sqlite_utils.cpp | 22 +++++----- .../storage/src/sync_able_engine.cpp | 3 +- 11 files changed, 48 insertions(+), 60 deletions(-) diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index 75c6d6167..ca7403d54 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -41,6 +41,11 @@ public: static constexpr int MAX_COMMIT_SIZE = 1000000; static constexpr int MAX_ENTRIES_SIZE = 1000000; + // In querySync, when getting query data finished, + // if the block size reach the half of max block size, will get deleted data next; + // if the block size not reach the half of max block size, will not get deleted data. + static constexpr float QUERY_SYNC_THRESHOLD = 0.50; + static constexpr uint64_t MAX_USER_ID_LENGTH = 128; static constexpr uint64_t MAX_APP_ID_LENGTH = 128; static constexpr uint64_t MAX_STORE_ID_LENGTH = 128; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp index a060a8279..f55194374 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -140,7 +140,7 @@ void DataTransformer::ReduceMapping(const std::vector &remoteFieldInf const std::vector &localFieldInfo, std::vector &indexMapping) { std::map fieldMap; - for (int i = 0; i < (int)remoteFieldInfo.size(); ++i) { + for (int i = 0; i < static_cast(remoteFieldInfo.size()); ++i) { const auto &fieldInfo = remoteFieldInfo[i]; fieldMap[fieldInfo.GetFieldName()] = i; } @@ -254,7 +254,7 @@ int SerializeBlobValue(const DataValue &dataValue, Parcel &parcel) if (errCode != E_OK) { return errCode; } - return parcel.WriteBlob(reinterpret_cast(val.GetData()), size); + return parcel.WriteBlob(reinterpret_cast(val.GetData()), size); } int DeSerializeBlobValue(DataValue &dataValue, Parcel &parcel) @@ -270,7 +270,7 @@ int DeSerializeBlobValue(DataValue &dataValue, Parcel &parcel) if (parcel.IsError()) { return -E_PARSE_FAIL; } - int errCode = val.WriteBlob(reinterpret_cast(array), blobLength); + int errCode = val.WriteBlob(reinterpret_cast(array), blobLength); if (errCode == E_OK) { dataValue = val; } @@ -360,7 +360,7 @@ int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalDa optionalData.push_back(std::nullopt); continue; } - if ((uint32_t)index >= valueList.size()) { + if (static_cast(index) >= valueList.size()) { return -E_INTERNAL_ERROR; // should not happen } optionalData.push_back(valueList[index]); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index e718b65bb..70418aa13 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -24,10 +24,6 @@ namespace DistributedDB { } \ } while (0) -namespace { -constexpr float QUERY_SYNC_THRESHOLD = 0.50; -} - RelationalSyncAbleStorage::RelationalSyncAbleStorage(StorageEngine *engine) : storageEngine_(static_cast(engine)) {} @@ -234,7 +230,7 @@ static bool CanHoldDeletedData(const std::vector &dataItems, const Dat bool reachThreshold = (dataItems.size() >= dataSizeInfo.packetSize); for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { blockSize += GetDataItemSerialSize(dataItems[i], appendLen); - reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); + reachThreshold = (blockSize >= dataSizeInfo.blockSize * DBConstant::QUERY_SYNC_THRESHOLD); } return !reachThreshold; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index e38138f0a..fea33f026 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -200,7 +200,7 @@ int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) break; } - syncEngine_ = std::make_shared(storageEngine_); + syncEngine_ = std::make_unique(storageEngine_); isInitialized_ = true; return E_OK; } while (false); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index dd0b50360..ebb3fb2b5 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -17,7 +17,7 @@ #ifdef RELATIONAL_STORE #include -#include +#include #include #include "irelational_store.h" @@ -66,7 +66,7 @@ private: int CleanDistributedDeviceTable(); // use for sync Interactive - std::shared_ptr syncEngine_ = nullptr; // For storage operate sync function + std::unique_ptr syncEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv RelationalSyncAbleStorage *storageEngine_ = nullptr; // For storage operate data SQLiteSingleRelationalStorageEngine *sqliteStorageEngine_ = nullptr; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h index 28ea03efb..7bb37460b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h @@ -35,14 +35,8 @@ protected: int CreateNewExecutor(bool isWrite, StorageExecutor *&handle) override; private: - // For executor. - - // For engine. - - // For db. int RegisterFunction(sqlite3 *db) const; }; } // namespace DistributedDB - #endif #endif // SQLITE_RELATIONAL_ENGINE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp index e0796c3d8..1110153a5 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp @@ -44,10 +44,6 @@ namespace DistributedDB { namespace { constexpr int DEF_LIFE_CYCLE_TIME = 60000; // 60s constexpr int WAIT_DELEGATE_CALLBACK_TIME = 100; - // In querySync, when getting query data finished, - // if the block size reach the half of max block size, will get deleted data next; - // if the block size not reach the half of max block size, will not get deleted data. - constexpr float QUERY_SYNC_THRESHOLD = 0.50; const std::string CREATE_DB_TIME = "createDBTime"; @@ -161,7 +157,7 @@ namespace { bool reachThreshold = false; for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { blockSize += SQLiteSingleVerStorageExecutor::GetDataItemSerialSize(dataItems[i], appendLen); - reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); + reachThreshold = (blockSize >= dataSizeInfo.blockSize * DBConstant::QUERY_SYNC_THRESHOLD); } return !reachThreshold; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp index 512a9a46e..6e6192733 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp @@ -84,9 +84,8 @@ int SQLiteSingleVerRelationalContinueToken::GetQuerySyncStatement(sqlite3 *db, s int SQLiteSingleVerRelationalContinueToken::GetDeletedDataStmt(sqlite3 *db, sqlite3_stmt *&stmt) const { // get stmt - int errCode = E_OK; const std::string sql = GetDeletedDataSQL(); - errCode = SQLiteUtils::GetStatement(db, sql, stmt); + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); if (errCode != E_OK) { goto ERROR; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 5e22a3e31..fd7657987 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -142,7 +142,7 @@ static int GetDataValueByType(sqlite3_stmt *statement, DataValue &value, Storage break; } case SQLITE3_TEXT: { - const char *colValue = reinterpret_cast(sqlite3_column_text(statement, cid)); + const char *colValue = reinterpret_cast(sqlite3_column_text(statement, cid)); if (colValue == nullptr) { value.ResetValue(); } else { @@ -218,28 +218,28 @@ static int BindDataValueByType(sqlite3_stmt *statement, const std::optional dev; - int errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 1, dev); + int errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 1, dev); // 1 means dev index if (errCode != E_OK) { return errCode; } logInfo.device = std::string(dev.begin(), dev.end()); std::vector oriDev; - errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 2, oriDev); + errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 2, oriDev); // 2 means ori_dev index if (errCode != E_OK) { return errCode; } logInfo.originDev = std::string(oriDev.begin(), oriDev.end()); - logInfo.timestamp = static_cast(sqlite3_column_int64(logStatement, 3)); - logInfo.wTimeStamp = static_cast(sqlite3_column_int64(logStatement, 4)); - logInfo.flag = static_cast(sqlite3_column_int64(logStatement, 5)); + logInfo.timestamp = static_cast(sqlite3_column_int64(logStatement, 3)); // 3 means timestamp index + logInfo.wTimeStamp = static_cast(sqlite3_column_int64(logStatement, 4)); // 4 means w_timestamp index + logInfo.flag = static_cast(sqlite3_column_int64(logStatement, 5)); // 5 means flag index logInfo.flag &= (~DataItem::LOCAL_FLAG); std::vector hashKey; - errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 6, hashKey); + errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 6, hashKey); // 6 means hashKey index if (errCode != E_OK) { return errCode; } @@ -298,13 +298,13 @@ int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Va goto ERROR; } - errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // BIND_KV_KEY_INDEX + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // 1 means key index if (errCode != E_OK) { LOGE("[SingleVerExe][BindPutKv]Bind key error:%d", errCode); goto ERROR; } - errCode = SQLiteUtils::BindBlobToStatement(statement, 2, value, true); // BIND_KV_VAL_INDEX + errCode = SQLiteUtils::BindBlobToStatement(statement, 2, value, true); // 2 means value index if (errCode != E_OK) { LOGE("[SingleVerExe][BindPutKv]Bind value error:%d", errCode); goto ERROR; @@ -509,17 +509,13 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen } // bind - SQLiteUtils::BindInt64ToStatement(statement, 1, dataKeyBind); - + SQLiteUtils::BindInt64ToStatement(statement, 1, dataKeyBind); // 1 means dataKey index std::vector originDev(logInfoBind.originDev.begin(), logInfoBind.originDev.end()); - SQLiteUtils::BindBlobToStatement(statement, 2, originDev); - - SQLiteUtils::BindInt64ToStatement(statement, 3, logInfoBind.timestamp); - SQLiteUtils::BindInt64ToStatement(statement, 4, logInfoBind.wTimeStamp); - SQLiteUtils::BindInt64ToStatement(statement, 5, logInfoBind.flag); - - SQLiteUtils::BindTextToStatement(statement, 6, logInfoBind.hashKey); - + SQLiteUtils::BindBlobToStatement(statement, 2, originDev); // 2 means ori_dev index + SQLiteUtils::BindInt64ToStatement(statement, 3, logInfoBind.timestamp); // 3 means timestamp index + SQLiteUtils::BindInt64ToStatement(statement, 4, logInfoBind.wTimeStamp); // 4 means w_timestamp index + SQLiteUtils::BindInt64ToStatement(statement, 5, logInfoBind.flag); // 5 means flag index + SQLiteUtils::BindTextToStatement(statement, 6, logInfoBind.hashKey); // 6 means hashKey index errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { return E_OK; @@ -661,10 +657,11 @@ int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *s std::vector fieldInfos; if (!isGettingDeletedData) { for (const auto &col: table_.GetFields()) { - LOGD("[GetDataItemForSync] field:%s type:%d cid:%d", col.second.GetFieldName().c_str(), - col.second.GetStorageType(), col.second.GetColumnId() + 7); + auto colType = col.second.GetStorageType(); + auto colId = col.second.GetColumnId() + 7; // 7 means the count of log table's column. + LOGD("[GetDataItemForSync] field:%s type:%d cid:%d", col.second.GetFieldName().c_str(), colType, colId); DataValue value; - errCode = GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId() + 7); + errCode = GetDataValueByType(stmt, value, colType, colId); if (errCode != E_OK) { return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 0132318c3..b4f6efaff 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -605,11 +605,11 @@ int AnalysisSchemaSqlAndTrigger(sqlite3 *db, const std::string &tableName, Table (void) SQLiteUtils::GetColumnTextValue(statement, 0, type); if (type == "table") { std::string createTableSql; - (void) SQLiteUtils::GetColumnTextValue(statement, 4, createTableSql); + (void) SQLiteUtils::GetColumnTextValue(statement, 4, createTableSql); // 4 means create table sql table.SetCreateTableSql(createTableSql); } else if (type == "trigger") { std::string triggerName; - (void) SQLiteUtils::GetColumnTextValue(statement, 1, triggerName); + (void) SQLiteUtils::GetColumnTextValue(statement, 1, triggerName); // 1 means trigger name table.AddTrigger(triggerName); } } else { @@ -639,9 +639,9 @@ int GetSchemaIndexList(sqlite3 *db, const std::string &tableName, std::vector(sqlite3_column_int64(statement, 3))); + field.SetNotNull(static_cast(sqlite3_column_int64(statement, 3))); // 3 means whether null index - (void) SQLiteUtils::GetColumnTextValue(statement, 4, tmpString); + (void) SQLiteUtils::GetColumnTextValue(statement, 4, tmpString); // 4 means default value index if (!tmpString.empty()) { field.SetDefaultValue(tmpString); } - if (sqlite3_column_int64(statement, 5)) { + if (sqlite3_column_int64(statement, 5)) { // 5 means primary key index table.SetPrimaryKey(field.GetFieldName()); } table.AddField(field); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp index f2ca51301..66c4735c4 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -21,7 +21,8 @@ namespace DistributedDB { SyncAbleEngine::SyncAbleEngine(ISyncInterface *store) - : started_(false), + : syncer_(), + started_(false), store_(store) {} -- Gitee