diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index c763dcedcbfdd239139cd06b64260be4f4029553..bd6a28f26eebccc0c89404a81dfdbde9b5cee24c 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -18,13 +18,16 @@ config("distrdb_config") { "include", "interfaces/include", "interfaces/src", + "interfaces/src/relational", "common/include", + "common/include/relational", "communicator/include", "storage/include", "storage/src", "storage/src/multiver", "storage/src/operation", "storage/src/sqlite", + "storage/src/sqlite/relational", "storage/src/upgrader", "syncer/include", "syncer/src", @@ -41,6 +44,7 @@ config("distrdb_config") { "USING_DB_JSON_EXTRACT_AUTOMATICALLY", "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", + "RELATIONAL_STORE", ] } @@ -48,6 +52,7 @@ config("distrdb_public_config") { visibility = [ "*:*" ] include_dirs = [ "interfaces/include", + "interfaces/include/relational", "include", ] } @@ -82,6 +87,7 @@ ohos_shared_library("distributeddb") { "common/src/query.cpp", "common/src/query_expression.cpp", "common/src/ref_object.cpp", + "common/src/relational/relational_schema_object.cpp", "common/src/runtime_context.cpp", "common/src/runtime_context_impl.cpp", "common/src/schema_object.cpp", @@ -115,6 +121,13 @@ ohos_shared_library("distributeddb") { "interfaces/src/kv_store_nb_delegate_impl.cpp", "interfaces/src/kv_store_result_set_impl.cpp", "interfaces/src/kv_store_snapshot_delegate_impl.cpp", + "interfaces/src/relational/data_value.cpp", + "interfaces/src/relational/relational_store_delegate_impl.cpp", + "interfaces/src/relational/relational_store_manager.cpp", + "interfaces/src/relational/relational_store_sqlite_ext.cpp", + "interfaces/src/relational/runtime_config.cpp", + "storage/src/data_transformer.cpp", + "storage/src/db_properties.cpp", "storage/src/default_factory.cpp", "storage/src/generic_kvdb.cpp", "storage/src/generic_kvdb_connection.cpp", @@ -145,10 +158,17 @@ ohos_shared_library("distributeddb") { "storage/src/operation/multi_ver_database_oper.cpp", "storage/src/operation/single_ver_database_oper.cpp", "storage/src/package_file.cpp", + "storage/src/relational_store_connection.cpp", + "storage/src/relational_store_instance.cpp", + "storage/src/relational_sync_able_storage.cpp", + "storage/src/relationaldb_properties.cpp", "storage/src/result_entries_window.cpp", "storage/src/single_ver_natural_store_commit_notify_data.cpp", "storage/src/sqlite/query_object.cpp", "storage/src/sqlite/query_sync_object.cpp", + "storage/src/sqlite/relational/sqlite_relational_store.cpp", + "storage/src/sqlite/relational/sqlite_relational_store_connection.cpp", + "storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp", "storage/src/sqlite/sqlite_local_kvdb.cpp", "storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -162,6 +182,8 @@ ohos_shared_library("distributeddb") { "storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp", + "storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "storage/src/sqlite/sqlite_single_ver_result_set.cpp", "storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", "storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", @@ -174,6 +196,7 @@ ohos_shared_library("distributeddb") { "storage/src/storage_engine.cpp", "storage/src/storage_engine_manager.cpp", "storage/src/storage_executor.cpp", + "storage/src/sync_able_engine.cpp", "storage/src/sync_able_kvdb.cpp", "storage/src/sync_able_kvdb_connection.cpp", "storage/src/upgrader/single_ver_database_upgrader.cpp", @@ -190,18 +213,17 @@ ohos_shared_library("distributeddb") { "syncer/src/multi_ver_sync_task_context.cpp", "syncer/src/multi_ver_syncer.cpp", "syncer/src/query_sync_water_mark_helper.cpp", + "syncer/src/single_ver_data_message_schedule.cpp", "syncer/src/single_ver_data_packet.cpp", "syncer/src/single_ver_data_sync.cpp", - "syncer/src/single_ver_data_sync_with_sliding_window.cpp", "syncer/src/single_ver_kv_syncer.cpp", + "syncer/src/single_ver_relational_syncer.cpp", "syncer/src/single_ver_serialize_manager.cpp", "syncer/src/single_ver_sync_engine.cpp", "syncer/src/single_ver_sync_state_machine.cpp", "syncer/src/single_ver_sync_target.cpp", "syncer/src/single_ver_sync_task_context.cpp", "syncer/src/single_ver_syncer.cpp", - "syncer/src/sliding_window_receiver.cpp", - "syncer/src/sliding_window_sender.cpp", "syncer/src/subscribe_manager.cpp", "syncer/src/sync_engine.cpp", "syncer/src/sync_operation.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index 99a6944b9081d6801617d966b1cb8d3b0c0158ce..ca7403d54a9b62296153b397995d59f70d6f89bb 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -41,6 +41,11 @@ public: static constexpr int MAX_COMMIT_SIZE = 1000000; static constexpr int MAX_ENTRIES_SIZE = 1000000; + // In querySync, when getting query data finished, + // if the block size reach the half of max block size, will get deleted data next; + // if the block size not reach the half of max block size, will not get deleted data. + static constexpr float QUERY_SYNC_THRESHOLD = 0.50; + static constexpr uint64_t MAX_USER_ID_LENGTH = 128; static constexpr uint64_t MAX_APP_ID_LENGTH = 128; static constexpr uint64_t MAX_STORE_ID_LENGTH = 128; @@ -92,6 +97,8 @@ public: static const std::string UPDATE_META_FUNC; + static const std::string SYSTEM_TABLE_PREFIX; + static constexpr uint32_t AUTO_SYNC_TIMEOUT = 5000; // 5s static constexpr uint32_t MANUAL_SYNC_TIMEOUT = 5000; // 5s @@ -109,6 +116,15 @@ public: static constexpr size_t MAX_SYNC_BLOCK_SIZE = 31457280; // 30MB static constexpr int DOUBLE_PRECISION = 15; + static constexpr int MAX_DISTRIBUTED_TABLE_COUNT = 32; + + static constexpr uint64_t MAX_LOG_SIZE_HIGH = 0x400000000ULL; // 16GB + static constexpr uint64_t MAX_LOG_SIZE_LOW = 0x400000ULL; // 4MB + static constexpr uint64_t MAX_LOG_SIZE_DEFAULT = 0x40000000ULL; // 1GB + + // For relational + static const std::string RELATIONAL_PREFIX; + static const std::string TIMESTAMP_ALIAS; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h index 5dd9a6e1fcb6c35721e30271ef252590329252e4..d76ab568fddca1b877c47a53c9e8b187757f45c9 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h @@ -115,10 +115,11 @@ constexpr int E_SYSTEM_API_ADAPTER_CALL_FAILED = (E_BASE + 91); // Adapter call constexpr int E_NOT_NEED_DELETE_MSG = (E_BASE + 92); // not need delete msg, will be delete by sliding window receiver constexpr int E_SLIDING_WINDOW_SENDER_ERR = (E_BASE + 93); // sliding window sender err constexpr int E_SLIDING_WINDOW_RECEIVER_INVALID_MSG = (E_BASE + 94); // sliding window receiver invalid msg -constexpr int E_IGNOR_DATA = (E_BASE + 95); // ignore the data changed by other devices and ignore the same data. +constexpr int E_IGNORE_DATA = (E_BASE + 95); // ignore the data changed by other devices and ignore the same data. constexpr int E_FORBID_CACHEDB = (E_BASE + 96); // such after rekey can not check passwd due to file control. constexpr int E_INTERCEPT_DATA_FAIL = (E_BASE + 97); // Intercept push data failed. constexpr int E_INVALID_COMPRESS_ALGO = (E_BASE + 98); // The algo is defined, but there's no implement for the algo. +constexpr int E_LOG_OVER_LIMITS = (E_BASE + 99); // The log file size is over the limits. // Num 150+ is reserved for schema related errno, since it may be added regularly constexpr int E_JSON_PARSE_FAIL = (E_BASE + 150); // Parse json fail in grammatical level constexpr int E_JSON_INSERT_PATH_EXIST = (E_BASE + 151); // Path already exist before insert diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_types.h b/services/distributeddataservice/libs/distributeddb/common/include/db_types.h index 697600aa8179561019f8988ddcd8b73d1447e96c..94f6cbd1c86d8c8a435f09842a3b9f22fafbcf77 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_types.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_types.h @@ -123,5 +123,16 @@ enum SingleVerConflictResolvePolicy { DEFAULT_LAST_WIN = 0, DENY_OTHER_DEV_AMEND_CUR_DEV_DATA = 1, }; + +struct SyncTimeRange { + TimeStamp beginTime = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp endTime = static_cast(INT64_MAX); + TimeStamp deleteEndTime = static_cast(INT64_MAX); + bool IsValid() const + { + return (beginTime <= endTime && deleteBeginTime <= deleteEndTime); + } +}; } // namespace DistributedDB #endif // DISTRIBUTEDDB_TYPES_H diff --git a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h index c9ecd26e6e7b1e79d01cf08151e9b1a207039285..ddad4bb797cc9e06032753286abf2a70f83c287e 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h @@ -65,6 +65,8 @@ public: int GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; int GetStringArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; + int GetObjectByFieldPath(const FieldPath &inPath, JsonObject &outObj) const; + // An empty fieldPath indicate the root, the outSubPath should be empty before call, we will not empty it at first. // If inPath is of multiple path, then outSubPath is combination of result of each inPath. int GetSubFieldPath(const FieldPath &inPath, std::set &outSubPath) const; diff --git a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h index 0e4c0dbc8df7b8cb418f397b79735159fa6725bd..ce07d65e1221028e0ebf35c1b134c264fab6e9ac 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h @@ -49,6 +49,8 @@ public: SchemaObject &schemaObject, std::string &canonicalDir); static uint8_t GetValidCompressionRate(uint8_t compressionRate); + + static bool CheckRelationalTableName(const std::string &tableName); }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index 47e4f99aa591388bd1d5c2d227d95c3d97f6a657..d13b4c1658296818ed7c27fd26f8b6bad7e35490 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -120,8 +120,8 @@ public: static RelationalSyncOpinion MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, const std::string &remoteSchema, uint8_t remoteSchemaType); - // The remoteOpinion.checkOnReceive is ignored + // The remoteOpinion.checkOnReceive is ignored static RelationalSyncStrategy ConcludeSyncStrategy(const RelationalSyncOpinion &localOpinion, const RelationalSyncOpinion &remoteOpinion); @@ -138,7 +138,7 @@ public: const std::map &GetTables() const; - const TableInfo &GetTable(const std::string& tableName) const; + TableInfo GetTable(const std::string& tableName) const; private: int CompareAgainstSchemaObject(const std::string &inSchemaString, std::map &cmpRst) const; @@ -153,6 +153,7 @@ private: int ParseCheckTableInfo(const JsonObject &inJsonObject); int ParseCheckTableName(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableDefine(const JsonObject &inJsonObject, TableInfo &resultTable); + int ParseCheckTableFieldInfo(const JsonObject &inJsonObject, const FieldPath &path, FieldInfo &table); int ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableIndex(const JsonObject &inJsonObject, TableInfo &resultTable); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp index e7c5a74ad63a078173cbd4cd2f810040e88ca3c7..6473420fddc3313edb0cf532c40d4aa55f8933fd 100644 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp @@ -22,7 +22,7 @@ namespace DistributedDB { DbAbility::DbAbility() { - for (auto & item : ABILITYBITS) { + for (const auto &item : ABILITYBITS) { dbAbilityItemSet_.insert(item); } dbAbility_.resize(ABILITYBITS.back().first + ABILITYBITS.back().second); @@ -135,11 +135,11 @@ uint8_t DbAbility::GetAbilityItem(const AbilityItem abilityType) const dbAbility_.size()); return 0; } - int skip = 0; - // dbAbility_ bit[0..len] : low-->high + uint32_t skip = 0; + // dbAbility_ bit[0..len] : low-->high, skip range 0..7 for (uint32_t pos = iter->first; pos < (iter->first + iter->second); pos++, skip++) { if (dbAbility_[pos]) { - data += dbAbility_[pos] << skip; + data += (static_cast(dbAbility_[pos])) << skip; } } } @@ -150,7 +150,7 @@ int DbAbility::SetAbilityItem(const AbilityItem &abilityType, uint8_t data) { auto iter = dbAbilityItemSet_.find(abilityType); if (iter != dbAbilityItemSet_.end()) { - if (data >= pow(2, iter->second)) { + if (data >= pow(2, iter->second)) { // 2: means binary LOGE("[DbAbility] value is invalid, data=%d, use_bit=%d", data, iter->second); return -E_INTERNAL_ERROR; } @@ -159,7 +159,7 @@ int DbAbility::SetAbilityItem(const AbilityItem &abilityType, uint8_t data) } int pos = iter->first; while (data) { - dbAbility_[pos] = data % 2; // means binary + dbAbility_[pos] = data % 2; // 2: means binary data = (data >> 1); pos++; } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp index de9192da34e253fc94c6d0364d84460f19db762e..c04b394063b5c0bffd71fc286d4dbb051f6bb093 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp @@ -290,9 +290,7 @@ std::string DBCommon::GenerateIdentifierId(const std::string &storeId, void DBCommon::SetDatabaseIds(KvDBProperties &properties, const std::string &appId, const std::string &userId, const std::string &storeId) { - properties.SetStringProp(KvDBProperties::APP_ID, appId); - properties.SetStringProp(KvDBProperties::USER_ID, userId); - properties.SetStringProp(KvDBProperties::STORE_ID, storeId); + properties.SetIdentifier(userId, appId, storeId); std::string oriStoreDir; std::string identifier = GenerateIdentifierId(storeId, appId, userId); if (properties.GetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, false)) { @@ -301,7 +299,6 @@ void DBCommon::SetDatabaseIds(KvDBProperties &properties, const std::string &app oriStoreDir = identifier; } std::string hashIdentifier = TransferHashString(identifier); - properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); std::string hashDir = TransferHashString(oriStoreDir); std::string hexHashDir = TransferStringToHex(hashDir); properties.SetStringProp(KvDBProperties::IDENTIFIER_DIR, hexHashDir); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp index 410e011373cf4b6ddf2b3b86f7539d2ecc5dc325..eb90c8a4a93eced1848ae0709995c13f90e3546b 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp @@ -63,4 +63,8 @@ const std::string DBConstant::TRIGGER_REFERENCES_NEW = "NEW."; const std::string DBConstant::TRIGGER_REFERENCES_OLD = "OLD."; const std::string DBConstant::UPDATE_META_FUNC = "update_meta_within_trigger"; + +const std::string DBConstant::SYSTEM_TABLE_PREFIX = "naturalbase_rdb_"; +const std::string DBConstant::RELATIONAL_PREFIX = SYSTEM_TABLE_PREFIX + "aux_"; +const std::string DBConstant::TIMESTAMP_ALIAS = RELATIONAL_PREFIX + "timestamp"; } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index 37a008ddbfc929a2018d09c2f3f6e1f5323ddd2e..7e9b2b470ac6b90db2dc1c30ddb9107db77dceea 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -519,7 +519,7 @@ int JsonObject::MoveToPath(const FieldPath &inPath, Json::Value *&exact, Json::V int JsonObject::InsertField(const FieldPath &inPath, const JsonObject &inValue, bool isAppend) { - if (inPath.empty() || inPath.size() > maxNestDepth_|| !inValue.IsValid()) { + if (inPath.empty() || inPath.size() > maxNestDepth_ || !inValue.IsValid()) { return -E_INVALID_ARGS; } if (!isValid_) { @@ -727,6 +727,7 @@ int JsonObject::GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const { if (!isValid_) { @@ -750,6 +771,7 @@ int JsonObject::GetStringArrayByFieldPath(const FieldPath &inPath, std::vector -#include #include "json_object.h" #include "relational_schema_object.h" +#include "schema_utils.h" namespace DistributedDB { const std::string &FieldInfo::GetFieldName() const @@ -97,15 +97,13 @@ void FieldInfo::SetColumnId(int cid) // return field define string like ("fieldName": "MY INT(21), NOT NULL, DEFAULT 123") std::string FieldInfo::ToAttributeString() const { - std::string attrStr = "\"" + fieldName_ + "\": \""; - attrStr += dataType_; - if (isNotNull_) { - attrStr += ", NOT NULL"; - } + std::string attrStr = "\"" + fieldName_ + "\": {"; + attrStr += "\"TYPE\":\"" + dataType_ + "\","; + attrStr += "\"NOT_NULL\":" + std::string(isNotNull_ ? "true" : "false") + ","; if (hasDefaultValue_) { - attrStr += ", " + defaultValue_; + attrStr += "\"DEFAULT\":\"" + defaultValue_ + "\""; } - attrStr += + "\""; + attrStr += "}"; return attrStr; } @@ -270,7 +268,7 @@ void TableInfo::SetDevId(const std::string &devId) } namespace { - std::string VectorJoin(const CompositeFields& fields, char con) + std::string VectorJoin(const CompositeFields &fields, char con) { std::string res; auto it = fields.begin(); @@ -292,13 +290,12 @@ JsonObject TableInfo::ToJsonObject() const tableJson.InsertField(FieldPath { "AUTOINCREMENT" }, FieldType::LEAF_FIELD_BOOL, jsonField); jsonField.stringValue = primaryKey_; tableJson.InsertField(FieldPath { "PRIMARY_KEY" }, FieldType::LEAF_FIELD_STRING, jsonField); - for (const auto& it : fields_) { + for (const auto &it : fields_) { jsonField.stringValue = it.second.ToAttributeString(); tableJson.InsertField(FieldPath { "DEFINE", it.first }, FieldType::LEAF_FIELD_STRING, jsonField); } - for (const auto& it : uniqueDefines_) { + for (const auto &it : uniqueDefines_) { jsonField.stringValue = VectorJoin(it, ','); - // TODO: add unique to tableJson } return tableJson; } @@ -337,7 +334,7 @@ int RelationalSyncOpinion::Deserialization(const Parcel &parcel) return E_OK; } -const SyncOpinion &RelationalSyncOpinion::GetTableOpinion(const std::string& tableName) const +const SyncOpinion &RelationalSyncOpinion::GetTableOpinion(const std::string &tableName) const { return opinions_.at(tableName); } @@ -408,12 +405,12 @@ int RelationalSchemaObject::ParseFromSchemaString(const std::string &inSchemaStr } schemaType_ = SchemaType::RELATIVE; - schemaString_ = inSchemaString; + schemaString_ = schemaObj.ToString(); isValid_ = true; return E_OK; } -void RelationalSchemaObject::AddRelationalTable(const TableInfo& tb) +void RelationalSchemaObject::AddRelationalTable(const TableInfo &tb) { tables_[tb.GetTableName()] = tb; } @@ -423,9 +420,13 @@ const std::map &RelationalSchemaObject::GetTables() cons return tables_; } -const TableInfo &RelationalSchemaObject::GetTable(const std::string& tableName) const +TableInfo RelationalSchemaObject::GetTable(const std::string &tableName) const { - return tables_.at(tableName); + auto it = tables_.find(tableName); + if (it != tables_.end()) { + return it->second; + } + return {}; } int RelationalSchemaObject::CompareAgainstSchemaObject(const std::string &inSchemaString, @@ -444,8 +445,9 @@ namespace { int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &fieldName, FieldType expectType, bool isNecessary, FieldValue &fieldValue) { - if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName}) && !isNecessary) { - return E_OK; + if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName})) { + LOGW("[RelationalSchema][Parse] Get schema %s not exist. isNecessary: %d", fieldName.c_str(), isNecessary); + return isNecessary ? -E_SCHEMA_PARSE_FAIL : -E_NOT_FOUND; } FieldType fieldType; @@ -456,11 +458,11 @@ int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &f } if (fieldType != expectType) { - LOGE("[RelationalSchema][Parse] Expect %s fieldType %d but : %d.", fieldName.c_str(), expectType, fieldType); + LOGE("[RelationalSchema][Parse] Expect %s fieldType %d but: %d.", fieldName.c_str(), expectType, fieldType); return -E_SCHEMA_PARSE_FAIL; } - errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {"NAME"}, fieldValue); + errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {fieldName}, fieldValue); if (errCode != E_OK) { LOGE("[RelationalSchema][Parse] Get schema %s value failed: %d.", fieldName.c_str(), errCode); return -E_SCHEMA_PARSE_FAIL; @@ -479,11 +481,7 @@ int RelationalSchemaObject::ParseRelationalSchema(const JsonObject &inJsonObject if (errCode != E_OK) { return errCode; } - errCode = ParseCheckSchemaTableDefine(inJsonObject); - if (errCode != E_OK) { - return errCode; - } - return E_OK; + return ParseCheckSchemaTableDefine(inJsonObject); } int RelationalSchemaObject::ParseCheckSchemaVersionMode(const JsonObject &inJsonObject) @@ -596,46 +594,67 @@ int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject } for (const auto &field : tableFields) { - if (field.second != FieldType::LEAF_FIELD_STRING) { - LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType STRING but : %s.", + if (field.second != FieldType::INTERNAL_FIELD_OBJECT) { + LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType INTERNAL OBJECT but : %s.", SchemaUtils::FieldTypeString(field.second).c_str()); return -E_SCHEMA_PARSE_FAIL; } - FieldValue fieldValue; - errCode = inJsonObject.GetFieldValueByFieldPath(field.first, fieldValue); - if (errCode != E_OK) { - LOGE("[RelationalSchema][Parse] Get schema TABLES DEFINE field value failed: %d.", errCode); - return -E_SCHEMA_PARSE_FAIL; - } - SchemaAttribute outAttr; - errCode = SchemaUtils::ParseAndCheckSchemaAttribute(fieldValue.stringValue, outAttr, false); + JsonObject fieldObj; + errCode = inJsonObject.GetObjectByFieldPath(field.first, fieldObj); if (errCode != E_OK) { - LOGE("[RelationalSchema][Parse] Parse schema TABLES DEFINE attribute failed: %d.", errCode); + LOGE("[RelationalSchema][Parse] Get table field object failed. %d", errCode); return errCode; } FieldInfo fieldInfo; - fieldInfo.SetFieldName(field.first[1]); - fieldInfo.SetDataType(outAttr.customFieldType); - fieldInfo.SetNotNull(outAttr.hasNotNullConstraint); - if (outAttr.hasDefaultValue) { - fieldInfo.SetDefaultValue(outAttr.defaultValue.stringValue); + fieldInfo.SetFieldName(field.first[0]); // 0 : first element in path + errCode = ParseCheckTableFieldInfo(fieldObj, field.first, fieldInfo); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Parse table field info failed. %d", errCode); + return -E_SCHEMA_PARSE_FAIL; } resultTable.AddField(fieldInfo); } return E_OK; } +int RelationalSchemaObject::ParseCheckTableFieldInfo(const JsonObject &inJsonObject, const FieldPath &path, + FieldInfo &table) +{ + FieldValue fieldValue; + int errCode = GetMemberFromJsonObject(inJsonObject, "TYPE", FieldType::LEAF_FIELD_STRING, true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + table.SetDataType(fieldValue.stringValue); + + errCode = GetMemberFromJsonObject(inJsonObject, "NOT_NULL", FieldType::LEAF_FIELD_BOOL, true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + table.SetNotNull(fieldValue.boolValue); + + errCode = GetMemberFromJsonObject(inJsonObject, "DEFAULT", FieldType::LEAF_FIELD_STRING, false, fieldValue); + if (errCode == E_OK) { + table.SetDefaultValue(fieldValue.stringValue); + } else if (errCode != -E_NOT_FOUND) { + return errCode; + } + + return E_OK; +} + int RelationalSchemaObject::ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable) { FieldValue fieldValue; - int errCode = GetMemberFromJsonObject(inJsonObject, "AUTOINCREMENT", FieldType::LEAF_FIELD_BOOL, - false, fieldValue); + int errCode = GetMemberFromJsonObject(inJsonObject, "AUTOINCREMENT", FieldType::LEAF_FIELD_BOOL, false, fieldValue); if (errCode == E_OK) { resultTable.SetAutoIncrement(fieldValue.boolValue); + } else if (errCode != -E_NOT_FOUND) { + return errCode; } - return errCode; + return E_OK; } int RelationalSchemaObject::ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable) @@ -681,7 +700,7 @@ int RelationalSchemaObject::ParseCheckTableIndex(const JsonObject &inJsonObject, LOGE("[RelationalSchema][Parse] Get schema TABLES INDEX field value failed: %d.", errCode); return -E_SCHEMA_PARSE_FAIL; } - resultTable.AddIndexDefine(field.first[1], indexDefine); + resultTable.AddIndexDefine(field.first[1], indexDefine); // 1 : second element in path } return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h index ea7376a1a1b723b7e90e26e992947027b3972f92..4fba674560c566bb89bd14f08f733953daf16149 100755 --- a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h +++ b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h @@ -46,6 +46,7 @@ struct AutoLaunchParam { std::string storeId; AutoLaunchOption option; AutoLaunchNotifier notifier; + std::string path; }; using AutoLaunchRequestCallback = std::function; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h index bd50513ae27efc01e8275eeb3cc8747b1277f078..ed5add66725a1e99b5ef765a90c2864c640f77b3 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h @@ -37,7 +37,10 @@ public: Blob(); ~Blob(); - DISABLE_COPY_ASSIGN_MOVE(Blob); + Blob(Blob &&); + Blob(const Blob &) = delete; + Blob &operator=(Blob &&) noexcept; + Blob &operator=(const Blob &) = delete; const uint8_t* GetData() const; uint32_t GetSize() const; @@ -65,6 +68,8 @@ public: DataValue &operator=(const double &doubleVal); DataValue &operator=(const Blob &blob); DataValue &operator=(const std::string &string); + int Set(Blob *&blob); + // equals bool operator==(const DataValue &dataValue) const; bool operator!=(const DataValue &dataValue) const; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 3e2923b77680b6f0b0bbd3b4eb638fb4b33078f9..fbf038564aa8476dc32b4d2dd0015ae3b7c0244e 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -27,23 +27,20 @@ public: DB_API virtual ~RelationalStoreDelegate() = default; struct Option { - bool createIfNecessary = true; + // split mode }; DB_API virtual DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) = 0; DB_API virtual DBStatus RemoveDeviceData(const std::string &device) = 0; - struct TableOption { - }; - - DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) = 0; + DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName) = 0; DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) = 0; DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait) = 0; + const Query &query, SyncStatusCallback &onComplete, bool wait) = 0; DB_API virtual DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) = 0; }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 17f669b0033f50eb6e232edfffdfd8709601a8c0..e491c1e7704afa30ca662c56043d7d7a22199673 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -21,8 +21,6 @@ #include "auto_launch_export.h" #include "relational_store_delegate.h" -#include "irelational_store.h" -#include "kvdb_properties.h" #include "types.h" namespace DistributedDB { @@ -36,16 +34,18 @@ public: RelationalStoreManager &operator=(const RelationalStoreManager &) = delete; RelationalStoreManager &operator=(RelationalStoreManager &&) = delete; - // PRAGMA journal_mode=WAL - // PRAGMA synchronous=FULL - // PRAGMA synchronous=NORMAL - DB_API void OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, - const std::function &callback); + DB_API DBStatus OpenStore(const std::string &path, const std::string &storeId, + const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate); DB_API DBStatus CloseStore(RelationalStoreDelegate *store); DB_API DBStatus DeleteStore(const std::string &path); + DB_API static void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback); + + DB_API static std::string GetRelationalStoreIdentifier(const std::string &userId, const std::string &appId, + const std::string &storeId); + private: std::string appId_; std::string userId_; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h index fb12ab170a7a1f206c361a3c0d3c0119485dbc13..8012f771d3078cdafec7c6a8a929c49487c8e653 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h @@ -56,6 +56,9 @@ enum DBStatus { SECURITY_OPTION_CHECK_ERROR, // such as remote device's SecurityOption not equal to local SCHEMA_VIOLATE_VALUE, // Values already exist in dbFile do not match new schema INTERCEPT_DATA_FAIL, // Interceptor push data failed. + LOG_OVER_LIMITS, // Log size is over the limits. + RELATIONAL_SCHEMA_NOT_FOUND, // the sync table is not a relational table + RELATIONAL_SCHEMA_CHANGED, // the schema was changed }; struct KvStoreConfig { @@ -79,6 +82,8 @@ enum PragmaCmd { RESULT_SET_CACHE_MODE, // Accept ResultSetCacheMode Type As PragmaData RESULT_SET_CACHE_MAX_SIZE, // Allowed Int Type Range [1,16], Unit MB SET_SYNC_RETRY, + SET_MAX_LOG_SIZE, + EXEC_CHECKPOINT, }; enum ResolutionPolicyType { diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp index aab890c2d8fcdb5949089a2b5910e1ee82a3d8f6..a0c0cde7c605527c830aac943e2355fefd342225 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp @@ -86,7 +86,9 @@ namespace { properties.SetIntProp(KvDBProperties::DATABASE_TYPE, KvDBProperties::SINGLE_VER_TYPE); properties.SetBoolProp(KvDBProperties::MEMORY_MODE, option.isMemoryDb); properties.SetBoolProp(KvDBProperties::ENCRYPTED_MODE, option.isEncryptedDb); - properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); + if (!option.isMemoryDb) { // memory db ignore store path + properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); + } properties.SetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, option.createDirByStoreIdOnly); properties.SetSchema(schema); properties.SetBoolProp(KvDBProperties::CHECK_INTEGRITY, option.isNeedIntegrityCheck); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp index b827ad0aecc9b18eb6578acf1de1a50459412ef9..492ea201197898a30ca755875632c9b67fd1a7fb 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp @@ -52,6 +52,7 @@ namespace { { -E_EKEYREVOKED, EKEYREVOKED_ERROR }, { -E_SECURITY_OPTION_CHECK_ERROR, SECURITY_OPTION_CHECK_ERROR }, { -E_INTERCEPT_DATA_FAIL, INTERCEPT_DATA_FAIL }, + { -E_LOG_OVER_LIMITS, LOG_OVER_LIMITS }, }; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp index f1f57a3863545bb64ce95ef8ae51d51b94d74c28..9c346ccdcca8cbdf0312758cd2b8686a6b15de2b 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp @@ -57,6 +57,8 @@ namespace { {RESULT_SET_CACHE_MODE, PRAGMA_RESULT_SET_CACHE_MODE}, {RESULT_SET_CACHE_MAX_SIZE, PRAGMA_RESULT_SET_CACHE_MAX_SIZE}, {SET_SYNC_RETRY, PRAGMA_SET_SYNC_RETRY}, + {SET_MAX_LOG_SIZE, PRAGMA_SET_MAX_LOG_SIZE}, + {EXEC_CHECKPOINT, PRAGMA_EXEC_CHECKPOINT}, }; const std::string INVALID_CONNECTION = "[KvStoreNbDelegate] Invalid connection for operation"; @@ -826,25 +828,10 @@ DBStatus KvStoreNbDelegateImpl::DeleteInner(const IOption &option, const Key &ke void KvStoreNbDelegateImpl::OnSyncComplete(const std::map &statuses, const std::function &devicesMap)> &onComplete) const { + const auto &statusMap = SyncOperation::DBStatusTransMap(); std::map result; for (const auto &pair : statuses) { DBStatus status = DB_ERROR; - static const std::map statusMap = { - { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, - { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, - { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, - { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, - { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, - { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, - { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, - { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, - { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, - { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, - { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, - { static_cast(SyncOperation::OP_INTERCEPT_DATA_FAIL), INTERCEPT_DATA_FAIL }, - { static_cast(SyncOperation::OP_MAX_LIMITS), OVER_MAX_LIMITS }, - { static_cast(SyncOperation::OP_INVALID_ARGS), INVALID_ARGS }, - }; auto iter = statusMap.find(pair.second); if (iter != statusMap.end()) { status = iter->second; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index ad5072181e94e7df2f36a12ac6d779775fc5fc78..96e70dd485d804dcf879de268d9012a319b9b578 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -33,6 +33,24 @@ Blob::~Blob() size_ = 0; } +Blob::Blob(Blob &&blob) : ptr_(blob.ptr_), size_(blob.size_) +{ + blob.ptr_ = nullptr; + blob.size_ = 0; +} + +Blob &Blob::operator=(Blob &&blob) noexcept +{ + if (this != &blob) { + delete[] ptr_; + ptr_ = blob.ptr_; + size_ = blob.size_; + blob.ptr_ = nullptr; + blob.size_ = 0; + } + return *this; +} + const uint8_t *Blob::GetData() const { return ptr_; @@ -45,8 +63,14 @@ uint32_t Blob::GetSize() const int Blob::WriteBlob(const uint8_t *ptrArray, const uint32_t &size) { - if (size == 0) return E_OK; - ptr_ = new(std::nothrow) uint8_t[size]; + if (ptrArray == nullptr || size == 0) { + return E_OK; + } + + delete[] ptr_; + ptr_ = nullptr; + + ptr_ = new (std::nothrow) uint8_t[size]; if (ptr_ == nullptr) { return -E_OUT_OF_MEMORY; } @@ -116,6 +140,10 @@ DataValue &DataValue::operator=(const DataValue &dataValue) DataValue &DataValue::operator=(DataValue &&dataValue) noexcept { + if (this == &dataValue) { + return *this; + } + ResetValue(); this->type_ = dataValue.type_; this->value_ = dataValue.value_; switch (type_) { @@ -159,7 +187,7 @@ DataValue &DataValue::operator=(const Blob &blob) if (blob.GetSize() <= 0) { return *this; } - value_.blobPtr = new(std::nothrow) Blob(); + value_.blobPtr = new (std::nothrow) Blob(); if (value_.blobPtr == nullptr) { return *this; } @@ -170,15 +198,28 @@ DataValue &DataValue::operator=(const Blob &blob) return *this; } +int DataValue::Set(Blob *&blob) +{ + ResetValue(); + if (blob == nullptr || blob->GetSize() <= 0) { + LOGE("Transfer Blob to DataValue failed."); + return -E_INVALID_ARGS; + } + type_ = StorageType::STORAGE_TYPE_BLOB; + value_.blobPtr = blob; + blob = nullptr; + return E_OK; +} + DataValue &DataValue::operator=(const std::string &string) { ResetValue(); - value_.blobPtr = new(std::nothrow) Blob(); + value_.blobPtr = new (std::nothrow) Blob(); if (value_.blobPtr == nullptr) { return *this; } type_ = StorageType::STORAGE_TYPE_TEXT; - value_.blobPtr->WriteBlob(reinterpret_cast(string.c_str()), string.size()); + value_.blobPtr->WriteBlob(reinterpret_cast(string.c_str()), string.size()); return *this; } @@ -212,7 +253,7 @@ bool DataValue::operator==(const DataValue &dataValue) const bool DataValue::operator!=(const DataValue &dataValue) const { - return !(*this==dataValue); + return !(*this == dataValue); } int DataValue::GetBool(bool &outVal) const @@ -247,7 +288,9 @@ int DataValue::GetBlob(Blob *&outVal) const if (type_ != StorageType::STORAGE_TYPE_BLOB && type_ != StorageType::STORAGE_TYPE_TEXT) { return -E_NOT_SUPPORT; } - outVal = new(std::nothrow) Blob(); + delete outVal; + outVal = nullptr; + outVal = new (std::nothrow) Blob(); if (outVal == nullptr) { return -E_OUT_OF_MEMORY; } @@ -371,8 +414,13 @@ int ObjectData::GetString(const std::string &fieldName, std::string &outValue) c if (errCode != E_OK) { return errCode; } + if (blob.GetSize() == 0) { + return errCode; + } outValue.resize(blob.GetSize()); - outValue.assign(blob.GetData(), blob.GetData() + blob.GetSize()); + if (blob.GetData() != nullptr) { + outValue.assign(blob.GetData(), blob.GetData() + blob.GetSize()); + } return errCode; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index fb4d08542149f6b884cfd04c2c6f95aa89023c56..a4d279cdf3c532b9f56f57b0d3de9be96d1ea439 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -16,8 +16,9 @@ #include "relational_store_delegate_impl.h" #include "db_errno.h" -#include "log_print.h" #include "kv_store_errno.h" +#include "log_print.h" +#include "param_check_utils.h" #include "sync_operation.h" namespace DistributedDB { @@ -52,15 +53,19 @@ DBStatus RelationalStoreDelegateImpl::RemoveDeviceData(const std::string &device return NOT_SUPPORT; } -DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName, const TableOption &option) +DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName) { - // check table Name and option + if (!ParamCheckUtils::CheckRelationalTableName(tableName)) { + LOGE("invalid table name."); + return INVALID_ARGS; + } + if (conn_ == nullptr) { LOGE("[RelationalStore Delegate] Invalid connection for operation!"); return DB_ERROR; } - int errCode = conn_->CreateDistributedTable(tableName, option); + int errCode = conn_->CreateDistributedTable(tableName); if (errCode != E_OK) { LOGW("[RelationalStore Delegate] Create Distributed table failed:%d", errCode); return TransferDBErrno(errCode); @@ -69,14 +74,15 @@ DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string & } DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait) + const Query &query, SyncStatusCallback &onComplete, bool wait) { if (conn_ == nullptr) { LOGE("Invalid connection for operation!"); return DB_ERROR; } - RelationalStoreConnection::SyncInfo syncInfo{devices, mode, onComplete, query, wait}; + RelationalStoreConnection::SyncInfo syncInfo{devices, mode, + std::bind(&RelationalStoreDelegateImpl::OnSyncComplete, std::placeholders::_1, onComplete), query, wait}; int errCode = conn_->SyncToDevice(syncInfo); if (errCode != E_OK) { LOGW("[RelationalStore Delegate] sync data to device failed:%d", errCode); @@ -115,5 +121,28 @@ void RelationalStoreDelegateImpl::SetReleaseFlag(bool flag) { releaseFlag_ = flag; } + +void RelationalStoreDelegateImpl::OnSyncComplete(const std::map> &devicesStatus, + SyncStatusCallback &onComplete) +{ + const auto &statusMap = SyncOperation::DBStatusTransMap(); + std::map> res; + for (const auto &[device, tablesStatus] : devicesStatus) { + for (const auto &tableStatus : tablesStatus) { + TableStatus table; + table.tableName = tableStatus.tableName; + DBStatus status = DB_ERROR; + auto iterator = statusMap.find(tableStatus.status); + if (iterator != statusMap.end()) { + status = iterator->second; + } + table.status = status; + res[device].push_back(table); + } + } + if (onComplete) { + onComplete(res); + } +} } // namespace DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 6dc5404c254ab98da64c2fe61ff2f078fcb349f6..76eb8defe22dff8f5b09518c5ed274a181b1e6aa 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -33,12 +33,14 @@ public: DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) override; + DBStatus Sync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait) override; + const Query &query, SyncStatusCallback &onComplete, bool wait) override; DBStatus RemoveDeviceData(const std::string &device) override; - DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) override; + DBStatus CreateDistributedTable(const std::string &tableName) override; + DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) override; // For connection @@ -46,6 +48,9 @@ public: void SetReleaseFlag(bool flag); private: + static void OnSyncComplete(const std::map> &devicesStatus, + SyncStatusCallback &onComplete); + RelationalStoreConnection *conn_ = nullptr; std::string storePath_; std::atomic releaseFlag_ = false; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h index 34316ef8260ab7d4c26fe4d75892f1d7c08742d1..5c24c1f2cf1b0824b39352380a113e18b2721bfa 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h @@ -20,7 +20,7 @@ #include #include "irelational_store.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" namespace DistributedDB { class RelationalStoreInstance final { @@ -28,19 +28,19 @@ public: RelationalStoreInstance(); ~RelationalStoreInstance() = default; - static RelationalStoreConnection *GetDatabaseConnection(const DBProperties &properties, int &errCode); + static RelationalStoreConnection *GetDatabaseConnection(const RelationalDBProperties &properties, int &errCode); static RelationalStoreInstance *GetInstance(); int CheckDatabaseFileStatus(const std::string &id); // public for test mock - static IRelationalStore *GetDataBase(const DBProperties &properties, int &errCode); + static IRelationalStore *GetDataBase(const RelationalDBProperties &properties, int &errCode); private: - IRelationalStore *OpenDatabase(const DBProperties &properties, int &errCode); + IRelationalStore *OpenDatabase(const RelationalDBProperties &properties, int &errCode); - void RemoveKvDBFromCache(const DBProperties &properties); - void SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties); + void RemoveKvDBFromCache(const RelationalDBProperties &properties); + void SaveKvDBToCache(IRelationalStore *store, const RelationalDBProperties &properties); static RelationalStoreInstance *instance_; static std::mutex instanceLock_; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 9a4a60324bc34d0cbdf3c274cbb73157e836b645..a72f5667a91668b6d85810b137bc3550eb6891a3 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -17,6 +17,7 @@ #include +#include "auto_launch.h" #include "relational_store_instance.h" #include "db_common.h" #include "param_check_utils.h" @@ -24,31 +25,28 @@ #include "db_errno.h" #include "kv_store_errno.h" #include "relational_store_delegate_impl.h" +#include "runtime_context.h" #include "platform_specific.h" namespace DistributedDB { -RelationalStoreManager::RelationalStoreManager(const std::string &appId, const std::string &userId) - : appId_(appId), - userId_(userId) -{} +namespace { +const int GET_CONNECT_RETRY = 3; +const int RETRY_GET_CONN_INTER = 30; -static void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, - const std::string &appId, const std::string &userId, DBProperties &properties) +void InitStoreProp(const std::string &storePath, const std::string &appId, const std::string &userId, + const std::string &storeId, RelationalDBProperties &properties) { - properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); - properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); - properties.SetStringProp(KvDBProperties::APP_ID, appId); - properties.SetStringProp(KvDBProperties::USER_ID, userId); - properties.SetStringProp(KvDBProperties::STORE_ID, storePath); // same as dir - std::string identifier = userId + "-" + appId + "-" + storePath; - std::string hashIdentifier = DBCommon::TransferHashString(identifier); - properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); + properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); + properties.SetIdentifier(userId, appId, storeId); +} } -static const int GET_CONNECT_RETRY = 3; -static const int RETRY_GET_CONN_INTER = 30; +RelationalStoreManager::RelationalStoreManager(const std::string &appId, const std::string &userId) + : appId_(appId), + userId_(userId) +{} -static RelationalStoreConnection *GetOneConnectionWithRetry(const DBProperties &properties, int &errCode) +static RelationalStoreConnection *GetOneConnectionWithRetry(const RelationalDBProperties &properties, int &errCode) { for (int i = 0; i < GET_CONNECT_RETRY; i++) { auto conn = RelationalStoreInstance::GetDatabaseConnection(properties, errCode); @@ -64,39 +62,38 @@ static RelationalStoreConnection *GetOneConnectionWithRetry(const DBProperties & return nullptr; } -void RelationalStoreManager::OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, - const std::function &callback) +DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const std::string &storeId, + const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate) { - if (!callback) { - LOGE("[KvStoreMgr] Invalid callback for kv store!"); - return; + if (delegate != nullptr) { + LOGE("[RelationalStoreMgr] Invalid delegate!"); + return INVALID_ARGS; } - if (!ParamCheckUtils::CheckStoreParameter("Relational_default_id", appId_, userId_) || path.empty()) { - callback(INVALID_ARGS, nullptr); - return; + std::string canonicalDir; + if (!ParamCheckUtils::CheckDataDir(path, canonicalDir)) { + return INVALID_ARGS; } - DBProperties properties; - InitStoreProp(option, path, appId_, userId_, properties); + if (!ParamCheckUtils::CheckStoreParameter(storeId, appId_, userId_) || path.empty()) { + return INVALID_ARGS; + } + + RelationalDBProperties properties; + InitStoreProp(canonicalDir, appId_, userId_, storeId, properties); int errCode = E_OK; auto *conn = GetOneConnectionWithRetry(properties, errCode); - DBStatus status = TransferDBErrno(errCode); if (conn == nullptr) { - callback(status, nullptr); - return; + return TransferDBErrno(errCode); } - auto store = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); - if (store == nullptr) { + delegate = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); + if (delegate == nullptr) { conn->Close(); - callback(DB_ERROR, nullptr); - return; + return DB_ERROR; } - - (void)conn->TriggerAutoSync(); - callback(OK, store); + return OK; } DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *store) @@ -177,5 +174,18 @@ DBStatus RelationalStoreManager::DeleteStore(const std::string &path) LOGE("Delete the kv store error:%d", errCode); return TransferDBErrno(errCode); } + +void RelationalStoreManager::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) +{ +} + +std::string RelationalStoreManager::GetRelationalStoreIdentifier(const std::string &userId, const std::string &appId, + const std::string &storeId) +{ + if (!ParamCheckUtils::CheckStoreParameter(storeId, appId, userId)) { + return ""; + } + return DBCommon::TransferHashString(DBCommon::GenerateIdentifierId(storeId, appId, userId)); +} } // namespace DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index 37ef57ada6909c8456de6080e71f83d7cdc63a29..e9e6562b4c671d80676734d5f2d97977c8120817 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -19,7 +19,7 @@ #include "relational_db_sync_interface.h" #include "sqlite_single_relational_storage_engine.h" -#include "sqlite_single_ver_continue_token.h" +#include "sqlite_single_ver_relational_continue_token.h" namespace DistributedDB { class RelationalSyncAbleStorage : public RelationalDBSyncInterface, public virtual RefObject { @@ -67,9 +67,6 @@ public: int GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; - // Release the continue token of getting data. - void ReleaseContinueToken(ContinueToken &continueStmtToken) const override; - int PutSyncDataWithQuery(const QueryObject &object, const std::vector &entries, const DeviceID &deviceName) override; @@ -112,7 +109,7 @@ private: int SetMaxTimeStamp(TimeStamp timestamp); // get - int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerContinueToken *&continueStmtToken, + int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerRelationalContinueToken *&token, const DataSizeSpecInfo &dataSizeInfo) const; // put diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h new file mode 100644 index 0000000000000000000000000000000000000000..215745a8eda24467da1c35cb9622fc367753dce9 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DB_PROPERTIES_H +#define DB_PROPERTIES_H + +#include +#include +#include + +namespace DistributedDB { +class DBProperties { +public: + // Get the string property according the name + std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; + + // Set the string property for the name + void SetStringProp(const std::string &name, const std::string &value); + + // Get the bool property according the name + bool GetBoolProp(const std::string &name, bool defaultValue) const; + + // Set the bool property for the name + void SetBoolProp(const std::string &name, bool value); + + // Get the bool property according the name + int GetIntProp(const std::string &name, int defaultValue) const; + + // Set the integer property for the name + void SetIntProp(const std::string &name, int value); + + // Set all indentifers + void SetIdentifier(const std::string &userId, const std::string &appId, const std::string &storeId); + + static const std::string CREATE_IF_NECESSARY; + static const std::string DATABASE_TYPE; + static const std::string DATA_DIR; + static const std::string USER_ID; + static const std::string APP_ID; + static const std::string STORE_ID; + static const std::string IDENTIFIER_DATA; + static const std::string IDENTIFIER_DIR; + +protected: + DBProperties() = default; + virtual ~DBProperties() = default; + + std::map stringProperties_; + std::map boolProperties_; + std::map intProperties_; +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h index 375823e39b9a306f1d624e693814b234ca52fda6..a6925c11d8662d22abbf9ea7e98951fd1318b1f3 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h @@ -22,17 +22,6 @@ #include "kvdb_properties.h" namespace DistributedDB { -struct SyncTimeRange { - TimeStamp beginTime = 0; - TimeStamp deleteBeginTime = 0; - TimeStamp endTime = static_cast(INT64_MAX); - TimeStamp deleteEndTime = static_cast(INT64_MAX); - bool IsValid() const - { - return (beginTime <= endTime && deleteBeginTime <= deleteEndTime); - } -}; - class ISyncInterface { public: enum { diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h index f873b5b165280796b4e1338f68983bb3e5841ae9..5759240efee9b17e49adc3ebf98c36d9e8090745 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h @@ -49,6 +49,8 @@ enum : int { PRAGMA_ADD_EQUAL_IDENTIFIER, PRAGMA_INTERCEPT_SYNC_DATA, PRAGMA_SUBSCRIBE_QUERY, + PRAGMA_SET_MAX_LOG_SIZE, + PRAGMA_EXEC_CHECKPOINT, }; struct PragmaSync { diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h index 6337094af09f61f30f7e164874fea1a525700b92..c985b0d832154b80f63d72c7d10f5298b3c4917f 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h @@ -20,41 +20,14 @@ #include #include +#include "db_properties.h" #include "schema_object.h" namespace DistributedDB { -class DBProperties { -public: - DBProperties() = default; - ~DBProperties() = default; - - // Get the string property according the name - std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; - - // Set the string property for the name - void SetStringProp(const std::string &name, const std::string &value); - - // Get the bool property according the name - bool GetBoolProp(const std::string &name, bool defaultValue) const; - - // Set the bool property for the name - void SetBoolProp(const std::string &name, bool value); - - // Get the bool property according the name - int GetIntProp(const std::string &name, int defaultValue) const; - - // Set the integer property for the name - void SetIntProp(const std::string &name, int value); -protected: - std::map stringProperties_; - std::map boolProperties_; - std::map intProperties_; -}; - class KvDBProperties final : public DBProperties { public: KvDBProperties(); - ~KvDBProperties(); + ~KvDBProperties() override; // Get the sub directory for different type database. static std::string GetStoreSubDirectory(int type); @@ -83,18 +56,10 @@ public: // The upper code will not change the schema if it is already set const SchemaObject &GetSchemaConstRef() const; - static const std::string CREATE_IF_NECESSARY; - static const std::string DATABASE_TYPE; - static const std::string DATA_DIR; - static const std::string USER_ID; - static const std::string APP_ID; - static const std::string STORE_ID; static const std::string FILE_NAME; static const std::string SYNC_MODE; static const std::string MEMORY_MODE; static const std::string ENCRYPTED_MODE; - static const std::string IDENTIFIER_DATA; - static const std::string IDENTIFIER_DIR; static const std::string FIRST_OPEN_IS_READ_ONLY; static const std::string CREATE_DIR_BY_STORE_ID_ONLY; static const std::string SECURITY_LABEL; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index e502357a79c436dd4b25aea0853765289aa27cde..f3dd78fc6c5dfe110fad022cac221d4076b67aa5 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -18,9 +18,11 @@ #include #include + +#include "db_types.h" #include "macro_utils.h" -#include "relational_store_delegate.h" #include "ref_object.h" +#include "relational_store_delegate.h" namespace DistributedDB { class IRelationalStore; @@ -30,15 +32,14 @@ public: struct SyncInfo { const std::vector &devices; SyncMode mode = SYNC_MODE_PUSH_PULL; - SyncStatusCallback &onComplete; + const SyncStatusCallback &onComplete; const Query &query; bool wait = true; }; - RelationalStoreConnection() = default; - explicit RelationalStoreConnection(IRelationalStore *store) - { - store_ = store; - }; + + RelationalStoreConnection(); + + explicit RelationalStoreConnection(IRelationalStore *store); virtual ~RelationalStoreConnection() = default; @@ -49,8 +50,8 @@ public: virtual int TriggerAutoSync() = 0; virtual int SyncToDevice(SyncInfo &info) = 0; virtual std::string GetIdentifier() = 0; - virtual int CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) = 0; + virtual int CreateDistributedTable(const std::string &tableName) = 0; + virtual int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) = 0; protected: // Get the stashed 'KvDB_ pointer' without ref. @@ -62,7 +63,7 @@ protected: virtual int Pragma(int cmd, void *parameter); IRelationalStore *store_ = nullptr; - std::atomic isExclusive_ = false; + std::atomic isExclusive_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h similarity index 53% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h rename to services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h index 23ad7d58a97e917e0510cf5b26ebda1bb965e2d0..4877e7220343a4d11111b67925624b4b7ed902c1 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h @@ -12,17 +12,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H -#define MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H +#ifndef RELATIONALDB_PROPERTIES_H +#define RELATIONALDB_PROPERTIES_H -#include -#include "single_ver_sync_task_context.h" +#include +#include +#include + +#include "db_properties.h" +#include "relational_schema_object.h" namespace DistributedDB { -class MockSingleVerSyncTaskContext : public SingleVerSyncTaskContext { +class RelationalDBProperties final : public DBProperties { public: - MOCK_METHOD1(HandleDataRequestRecv, int(const Message *)); - MOCK_CONST_METHOD0(IsReceiveWaterMarkErr, bool(void)); + RelationalDBProperties(); + ~RelationalDBProperties() override; + + // is schema exist + bool IsSchemaExist() const; + + // set schema + void SetSchema(const RelationalSchemaObject &schema); + + // get schema + RelationalSchemaObject GetSchema() const; + +private: + RelationalSchemaObject schema_; }; -} // namespace DistributedDB -#endif // MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp index c4e020261e3acc597b205257197520cb772ff2a5..f55194374d9d32125e4aa18dd1ca17d1e9f0fbed 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -118,17 +118,18 @@ uint32_t DataTransformer::CalDataValueLength(const DataValue &dataValue) return 0u; } uint32_t length = 0; - std::string str; switch (dataValue.GetType()) { case StorageType::STORAGE_TYPE_BLOB: (void)dataValue.GetBlobLength(length); length = Parcel::GetEightByteAlign(length); length += Parcel::GetUInt32Len(); // record data length break; - case StorageType::STORAGE_TYPE_TEXT: + case StorageType::STORAGE_TYPE_TEXT: { + std::string str; (void)dataValue.GetText(str); length = Parcel::GetStringLen(str); break; + } default: break; } @@ -139,7 +140,7 @@ void DataTransformer::ReduceMapping(const std::vector &remoteFieldInf const std::vector &localFieldInfo, std::vector &indexMapping) { std::map fieldMap; - for (int i = 0; i < (int)remoteFieldInfo.size(); ++i) { + for (int i = 0; i < static_cast(remoteFieldInfo.size()); ++i) { const auto &fieldInfo = remoteFieldInfo[i]; fieldMap[fieldInfo.GetFieldName()] = i; } @@ -245,7 +246,7 @@ int SerializeBlobValue(const DataValue &dataValue, Parcel &parcel) { Blob val; (void)dataValue.GetBlob(val); - const uint32_t &size = val.GetSize(); + uint32_t size = val.GetSize(); if (size == 0) { return SerializeNullValue(dataValue, parcel); } @@ -253,7 +254,7 @@ int SerializeBlobValue(const DataValue &dataValue, Parcel &parcel) if (errCode != E_OK) { return errCode; } - return parcel.WriteBlob(reinterpret_cast(val.GetData()), size); + return parcel.WriteBlob(reinterpret_cast(val.GetData()), size); } int DeSerializeBlobValue(DataValue &dataValue, Parcel &parcel) @@ -269,7 +270,7 @@ int DeSerializeBlobValue(DataValue &dataValue, Parcel &parcel) if (parcel.IsError()) { return -E_PARSE_FAIL; } - int errCode = val.WriteBlob(reinterpret_cast(array), blobLength); + int errCode = val.WriteBlob(reinterpret_cast(array), blobLength); if (errCode == E_OK) { dataValue = val; } @@ -301,27 +302,22 @@ int DataTransformer::SerializeValue(Value &value, const RowData &rowData, const uint32_t totalLength = Parcel::GetUInt64Len(); // first record field count for (uint32_t i = 0; i < rowData.size(); ++i) { const auto &dataValue = rowData[i]; - const auto &fieldInfo = fieldInfoList[i]; - if (dataValue.GetType() != StorageType::STORAGE_TYPE_NULL && - dataValue.GetType() != fieldInfo.GetStorageType()) { - return -E_INVALID_DATA; - } if (typeFuncMap.find(dataValue.GetType()) == typeFuncMap.end()) { return -E_NOT_SUPPORT; } + totalLength += Parcel::GetUInt32Len(); // For save the dataValue's type. uint32_t dataLength = CalDataValueLength(dataValue); totalLength += dataLength; } value.resize(totalLength); + if (value.size() != totalLength) { + return -E_OUT_OF_MEMORY; + } Parcel parcel(value.data(), value.size()); (void)parcel.WriteUInt64(rowData.size()); - int index = 0; for (const auto &dataValue : rowData) { - const auto &fieldInfo = fieldInfoList[index++]; StorageType type = dataValue.GetType(); - if (dataValue.GetType() == StorageType::STORAGE_TYPE_NULL) { - type = fieldInfo.GetStorageType(); - } + parcel.WriteUInt32(static_cast(type)); int errCode = typeFuncMap[type].serializeFunc(dataValue, parcel); if (errCode != E_OK) { value.clear(); @@ -346,7 +342,13 @@ int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalDa DataValue dataValue; LOGD("[DataTransformer][DeSerializeValue] start deSerialize %s type %d", fieldInfo.GetFieldName().c_str(), fieldInfo.GetStorageType()); - int errCode = typeFuncMap[fieldInfo.GetStorageType()].deSerializeFunc(dataValue, parcel); + uint32_t type = 0; + parcel.ReadUInt32(type); + auto iter = typeFuncMap.find(static_cast(type)); + if (iter == typeFuncMap.end()) { + return -E_PARSE_FAIL; + } + int errCode = iter->second.deSerializeFunc(dataValue, parcel); if (errCode != E_OK) { LOGD("[DataTransformer][DeSerializeValue] deSerialize %s failed", fieldInfo.GetFieldName().c_str()); return errCode; @@ -358,24 +360,23 @@ int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalDa optionalData.push_back(std::nullopt); continue; } - if ((uint32_t)index >= valueList.size()) { + if (static_cast(index) >= valueList.size()) { return -E_INTERNAL_ERROR; // should not happen } - if (valueList[index].GetType() == StorageType::STORAGE_TYPE_NULL) { - optionalData.push_back(std::nullopt); - } else { - optionalData.push_back(valueList[index]); - } + optionalData.push_back(valueList[index]); } return E_OK; } int DataTransformer::SerializeHashKey(Key &key, const std::string &hashKey) { - key.resize(Parcel::GetStringLen(hashKey), 0); - Parcel parcel(key.data(), Parcel::GetStringLen(hashKey)); - int errCode = parcel.WriteString(hashKey); - return errCode; + uint32_t len = Parcel::GetStringLen(hashKey); + key.resize(len, 0); + if (key.size() != len) { + return -E_OUT_OF_MEMORY; + } + Parcel parcel(key.data(), len); + return parcel.WriteString(hashKey); } int DataTransformer::DeSerializeHashKey(const Key &key, std::string &hashKey) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d6920dcc8cd16aa64f907a0f63f55e4626acfe58 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "db_common.h" +#include "db_properties.h" + +namespace DistributedDB { +const std::string DBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; +const std::string DBProperties::DATABASE_TYPE = "databaseType"; +const std::string DBProperties::DATA_DIR = "dataDir"; +const std::string DBProperties::USER_ID = "userId"; +const std::string DBProperties::APP_ID = "appId"; +const std::string DBProperties::STORE_ID = "storeId"; +const std::string DBProperties::IDENTIFIER_DATA = "identifier"; +const std::string DBProperties::IDENTIFIER_DIR = "identifierDir"; + +std::string DBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const +{ + auto iter = stringProperties_.find(name); + if (iter != stringProperties_.end()) { + return iter->second; + } + return defaultValue; +} + +void DBProperties::SetStringProp(const std::string &name, const std::string &value) +{ + stringProperties_[name] = value; +} + +bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const +{ + auto iter = boolProperties_.find(name); + if (iter != boolProperties_.end()) { + return iter->second; + } + return defaultValue; +} + +void DBProperties::SetBoolProp(const std::string &name, bool value) +{ + boolProperties_[name] = value; +} + +int DBProperties::GetIntProp(const std::string &name, int defaultValue) const +{ + auto iter = intProperties_.find(name); + if (iter != intProperties_.end()) { + return iter->second; + } + return defaultValue; +} + +void DBProperties::SetIntProp(const std::string &name, int value) +{ + intProperties_[name] = value; +} + +void DBProperties::SetIdentifier(const std::string &userId, const std::string &appId, const std::string &storeId) +{ + SetStringProp(DBProperties::APP_ID, appId); + SetStringProp(DBProperties::USER_ID, userId); + SetStringProp(DBProperties::STORE_ID, storeId); + std::string hashIdentifier = DBCommon::TransferHashString(DBCommon::GenerateIdentifierId(storeId, appId, userId)); + SetStringProp(DBProperties::IDENTIFIER_DATA, hashIdentifier); +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h index 2d8cf45e0f67e7d606522b3f9efb210e5d315e04..a9714e556139794d230c69c96d84cf45007b7a37 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h @@ -20,7 +20,7 @@ #include #include "ref_object.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" #include "relational_store_connection.h" namespace DistributedDB { @@ -31,7 +31,7 @@ public: DISABLE_COPY_ASSIGN_MOVE(IRelationalStore); // Open the database. - virtual int Open(const DBProperties &kvDBProp) = 0; + virtual int Open(const RelationalDBProperties &properties) = 0; virtual void WakeUpSyncer() = 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp index ee95611ea9f25a4c0e7e85b499b9a7321a324109..c10569d8213c9230958c4ff4719f5b1e9531942e 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp @@ -18,17 +18,9 @@ #include "db_constant.h" namespace DistributedDB { -const std::string KvDBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; -const std::string KvDBProperties::DATABASE_TYPE = "databaseType"; -const std::string KvDBProperties::DATA_DIR = "dataDir"; -const std::string KvDBProperties::USER_ID = "userId"; -const std::string KvDBProperties::APP_ID = "appId"; -const std::string KvDBProperties::STORE_ID = "storeId"; const std::string KvDBProperties::FILE_NAME = "fileName"; const std::string KvDBProperties::MEMORY_MODE = "memoryMode"; const std::string KvDBProperties::ENCRYPTED_MODE = "isEncryptedDb"; -const std::string KvDBProperties::IDENTIFIER_DATA = "identifier"; -const std::string KvDBProperties::IDENTIFIER_DIR = "identifierDir"; const std::string KvDBProperties::FIRST_OPEN_IS_READ_ONLY = "firstOpenIsReadOnly"; const std::string KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY = "createDirByStoreIdOnly"; const std::string KvDBProperties::SECURITY_LABEL = "securityLabel"; @@ -59,51 +51,6 @@ std::string KvDBProperties::GetStoreSubDirectory(int type) } } -std::string DBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const -{ - auto iter = stringProperties_.find(name); - if (iter != stringProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetStringProp(const std::string &name, const std::string &value) -{ - stringProperties_[name] = value; -} - -bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const -{ - auto iter = boolProperties_.find(name); - if (iter != boolProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetBoolProp(const std::string &name, bool value) -{ - boolProperties_[name] = value; -} - -int DBProperties::GetIntProp(const std::string &name, int defaultValue) const -{ - auto iter = intProperties_.find(name); - if (iter != intProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetIntProp(const std::string &name, int value) -{ - intProperties_[name] = value; -} - void KvDBProperties::GetPassword(CipherType &type, CipherPassword &password) const { type = cipherType_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp index 4fc241f370cb7f2491f2662f268878c0b745df8b..c04d81193a1bc286808972e6c9134d385766768a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp @@ -18,6 +18,13 @@ #include "sqlite_single_ver_relational_storage_executor.h" namespace DistributedDB { +RelationalStoreConnection::RelationalStoreConnection() : isExclusive_(false) +{} + +RelationalStoreConnection::RelationalStoreConnection(IRelationalStore *store) + : store_(store), isExclusive_(false) +{} + int RelationalStoreConnection::Pragma(int cmd, void *parameter) { return E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp index e48bc8230e24d145e283748d8e9e40b23f8c1f9c..366d78863fcc80e252f65c8a39afbdd356227c71 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp @@ -18,8 +18,9 @@ #include #include -#include "sqlite_relational_store.h" +#include "db_common.h" #include "db_errno.h" +#include "sqlite_relational_store.h" #include "log_print.h" namespace DistributedDB { @@ -54,10 +55,10 @@ int RelationalStoreInstance::CheckDatabaseFileStatus(const std::string &id) return E_OK; } -static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCode) +static IRelationalStore *GetFromCache(const RelationalDBProperties &properties, int &errCode) { errCode = E_OK; - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); auto iter = dbs_.find(identifier); if (iter == dbs_.end()) { errCode = -E_NOT_FOUND; @@ -65,7 +66,6 @@ static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCo } auto *db = iter->second; - if (db == nullptr) { LOGE("Store cache is nullptr, there may be a logic error"); errCode = -E_INTERNAL_ERROR; @@ -76,21 +76,21 @@ static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCo } // Save to IKvDB to the global map -void RelationalStoreInstance::RemoveKvDBFromCache(const DBProperties &properties) +void RelationalStoreInstance::RemoveKvDBFromCache(const RelationalDBProperties &properties) { std::lock_guard lockGuard(storeLock_); - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); dbs_.erase(identifier); } -void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties) +void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const RelationalDBProperties &properties) { if (store == nullptr) { return; } { - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); store->WakeUpSyncer(); if (dbs_.count(identifier) == 0) { dbs_.insert(std::pair(identifier, store)); @@ -98,11 +98,11 @@ void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const DBP } } -IRelationalStore *RelationalStoreInstance::OpenDatabase(const DBProperties &properties, int &errCode) +IRelationalStore *RelationalStoreInstance::OpenDatabase(const RelationalDBProperties &properties, int &errCode) { auto db = new (std::nothrow) SQLiteRelationalStore(); if (db == nullptr) { - LOGE("Failed to get IKvDB! err:%d", errCode); + LOGE("Failed to get relational store! err:%d", errCode); return nullptr; } @@ -122,7 +122,7 @@ IRelationalStore *RelationalStoreInstance::OpenDatabase(const DBProperties &prop return db; } -IRelationalStore *RelationalStoreInstance::GetDataBase(const DBProperties &properties, int &errCode) +IRelationalStore *RelationalStoreInstance::GetDataBase(const RelationalDBProperties &properties, int &errCode) { std::lock_guard lockGuard(storeLock_); auto *db = GetFromCache(properties, errCode); @@ -145,8 +145,11 @@ IRelationalStore *RelationalStoreInstance::GetDataBase(const DBProperties &prope return db; } -RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const DBProperties &properties, int &errCode) +RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const RelationalDBProperties &properties, + int &errCode) { + std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + LOGD("Begin to get [%s] database connection.", STR_MASK(DBCommon::TransferStringToHex(identifier))); IRelationalStore *db = GetDataBase(properties, errCode); if (db == nullptr) { LOGE("Failed to open the db:%d", errCode); @@ -157,9 +160,8 @@ RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const if (connection == nullptr) { // not kill db, Other operations like import may be used concurrently LOGE("Failed to get the db connect for delegate:%d", errCode); } - RefObject::DecObjRef(db); // restore the reference increased by the cache. - // kvDB = nullptr; + RefObject::DecObjRef(db); // restore the reference increased by the cache. return connection; } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index c8082bad6ac946c1421d1f5bd24f224a3bcc3ec2..70418aa13a454d9b9e4f0e6d094bd8d88a0da9f3 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -196,15 +196,6 @@ const KvDBProperties &RelationalSyncAbleStorage::GetDbProperties() const return properties; } -static void ReleaseKvEntries(std::vector &entries) -{ - for (auto &itemEntry : entries) { - delete itemEntry; - itemEntry = nullptr; - } - entries.clear(); -} - static int GetKvEntriesByDataItems(std::vector &entries, std::vector &dataItems) { int errCode = E_OK; @@ -213,7 +204,7 @@ static int GetKvEntriesByDataItems(std::vector &entries, std if (entry == nullptr) { errCode = -E_OUT_OF_MEMORY; LOGE("GetKvEntries failed, errCode:%d", errCode); - ReleaseKvEntries(entries); + SingleVerKvEntry::Release(entries); break; } entry->SetEntryData(std::move(item)); @@ -233,20 +224,19 @@ static size_t GetDataItemSerialSize(const DataItem &item, size_t appendLen) return dataSize; } -static constexpr float QUERY_SYNC_THRESHOLD = 0.50; static bool CanHoldDeletedData(const std::vector &dataItems, const DataSizeSpecInfo &dataSizeInfo, size_t appendLen) { - bool reachThreshold = false; + bool reachThreshold = (dataItems.size() >= dataSizeInfo.packetSize); for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { blockSize += GetDataItemSerialSize(dataItems[i], appendLen); - reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); + reachThreshold = (blockSize >= dataSizeInfo.blockSize * DBConstant::QUERY_SYNC_THRESHOLD); } return !reachThreshold; } static void ProcessContinueTokenForQuerySync(const std::vector &dataItems, int &errCode, - SQLiteSingleVerContinueToken *&token) + SQLiteSingleVerRelationalContinueToken *&token) { if (errCode != -E_UNFINISHED) { // Error happened or get data finished. Token should be cleared. delete token; @@ -261,26 +251,15 @@ static void ProcessContinueTokenForQuerySync(const std::vector &dataIt token = nullptr; return; } - - TimeStamp nextBeginTime = dataItems.back().timeStamp + 1; - if (nextBeginTime > INT64_MAX) { - nextBeginTime = INT64_MAX; - } - bool getDeleteData = ((dataItems.back().flag & DataItem::DELETE_FLAG) != 0); - if (getDeleteData) { - token->FinishGetQueryData(); - token->SetDeletedNextBeginTime("", nextBeginTime); - } else { - token->SetNextBeginTime("", nextBeginTime); - } + token->SetNextBeginTime(dataItems.back()); } /** - * Caller must ensure that parameter continueStmtToken is valid. + * Caller must ensure that parameter token is valid. * If error happened, token will be deleted here. */ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &dataItems, - SQLiteSingleVerContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const + SQLiteSingleVerRelationalContinueToken *&token, const DataSizeSpecInfo &dataSizeInfo) const { if (storageEngine_ == nullptr) { return -E_INVALID_DB; @@ -293,47 +272,29 @@ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &da goto ERROR; } - errCode = handle->SetTableInfo(continueStmtToken->GetQuery()); + errCode = handle->SetTableInfo(token->GetQuery()); if (errCode != E_OK) { goto ERROR; } - // Get query data. - if (!continueStmtToken->IsGetQueryDataFinished()) { - LOGD("[SingleVerNStore] Get query data between %llu and %llu.", continueStmtToken->GetQueryBeginTime(), - continueStmtToken->GetQueryEndTime()); - errCode = handle->GetSyncDataByQuery( - dataItems, Parcel::GetAppendedLen(), continueStmtToken->GetQuery(), dataSizeInfo, - // need modify - std::make_pair(continueStmtToken->GetQueryBeginTime(), continueStmtToken->GetQueryEndTime())); - } - - // Get query data finished. - if (errCode == E_OK || errCode == -E_FINISHED) { - // Clear query timeRange. - continueStmtToken->FinishGetQueryData(); - - // Get delete time next. - if (!continueStmtToken->IsGetDeletedDataFinished() && - CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())) { - LOGD("[SingleVerNStore] Get deleted data between %llu and %llu.", continueStmtToken->GetDeletedBeginTime(), - continueStmtToken->GetDeletedEndTime()); - errCode = handle->GetDeletedSyncDataByTimestamp( - dataItems, Parcel::GetAppendedLen(), - // need modify - continueStmtToken->GetDeletedBeginTime(), continueStmtToken->GetDeletedEndTime(), dataSizeInfo); + do { + errCode = handle->GetSyncDataByQuery(dataItems, + Parcel::GetAppendedLen(), + dataSizeInfo, + std::bind(&SQLiteSingleVerRelationalContinueToken::GetStatement, *token, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + if (errCode == -E_FINISHED) { + token->FinishGetData(); + errCode = token->IsGetAllDataFinished() ? E_OK : -E_UNFINISHED; } - } - - if (errCode == -E_FINISHED) { - errCode = E_OK; - } + } while (errCode == -E_UNFINISHED && + CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())); ERROR: if (errCode != -E_UNFINISHED && errCode != E_OK) { // Error happened. dataItems.clear(); } - ProcessContinueTokenForQuerySync(dataItems, errCode, continueStmtToken); + ProcessContinueTokenForQuerySync(dataItems, errCode, token); ReleaseHandle(handle); return errCode; } @@ -348,13 +309,24 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return -E_INVALID_ARGS; } - auto token = new (std::nothrow) SQLiteSingleVerContinueToken(timeRange, query); // release in sync module + auto token = new (std::nothrow) SQLiteSingleVerRelationalContinueToken(timeRange, query); if (token == nullptr) { LOGE("[SingleVerNStore] Allocate continue token failed."); return -E_OUT_OF_MEMORY; } - int innerCode; + continueStmtToken = static_cast(token); + return GetSyncDataNext(entries, continueStmtToken, dataSizeInfo); +} + +int RelationalSyncAbleStorage::GetSyncDataNext(std::vector &entries, + ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const +{ + auto token = static_cast(continueStmtToken); + if (!token->CheckValid()) { + return -E_INVALID_ARGS; + } + std::vector dataItems; int errCode = GetSyncDataForQuerySync(dataItems, token, dataSizeInfo); if (errCode != E_OK && errCode != -E_UNFINISHED) { // The code need be sent to outside except new error happened. @@ -362,7 +334,7 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return errCode; } - innerCode = GetKvEntriesByDataItems(entries, dataItems); + int innerCode = GetKvEntriesByDataItems(entries, dataItems); if (innerCode != E_OK) { errCode = innerCode; delete token; @@ -372,18 +344,6 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return errCode; } -int RelationalSyncAbleStorage::GetSyncDataNext(std::vector &entries, - ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const -{ - return E_OK; -} - -// Release the continue token of getting data. -void RelationalSyncAbleStorage::ReleaseContinueToken(ContinueToken &continueStmtToken) const -{ - return; -} - int RelationalSyncAbleStorage::PutSyncDataWithQuery(const QueryObject &object, const std::vector &entries, const DeviceID &deviceName) { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp similarity index 59% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h rename to services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp index 5bd453f601979ee076bf2b2f9c2135c3ba088225..3a87aafb5e5a8a7d829fdc8b3819b806409656e8 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp @@ -12,17 +12,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MOCK_SINGLE_VER_DATA_SYNC_H -#define MOCK_SINGLE_VER_DATA_SYNC_H -#include -#include "single_ver_data_sync.h" +#include "relationaldb_properties.h" namespace DistributedDB { -class MockSingleVerDataSync : public SingleVerDataSync { -public: - MOCK_METHOD4(SendDataAck, int(SingleVerSyncTaskContext *, const Message *, int32_t, WaterMark)); -}; -} // namespace DistributedDB +RelationalDBProperties::RelationalDBProperties() +{} -#endif // MOCK_SINGLE_VER_DATA_SYNC_H +RelationalDBProperties::~RelationalDBProperties() +{} + +bool RelationalDBProperties::IsSchemaExist() const +{ + return schema_.IsSchemaValid(); +} + +void RelationalDBProperties::SetSchema(const RelationalSchemaObject &schema) +{ + schema_ = schema; +} + +RelationalSchemaObject RelationalDBProperties::GetSchema() const +{ + return schema_; +} +} diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index ed7bdb058df387637c4cb5f97ec47881e8c10d46..fea33f026b00ad529322fd89f2a0b856f7ed62e9 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -15,18 +15,26 @@ #ifdef RELATIONAL_STORE #include "sqlite_relational_store.h" +#include "db_common.h" #include "db_errno.h" #include "log_print.h" #include "db_types.h" #include "sqlite_relational_store_connection.h" +#include "storage_engine_manager.h" namespace DistributedDB { +namespace { + constexpr const char *RELATIONAL_SCHEMA_KEY = "relational_schema"; + constexpr const char *LOG_TABLE_VERSION_KEY = "log_table_versoin"; + constexpr const char *LOG_TABLE_VERSION_1 = "1.0"; +} + SQLiteRelationalStore::~SQLiteRelationalStore() { delete sqliteStorageEngine_; + sqliteStorageEngine_ = nullptr; } - // Called when a new connection created. void SQLiteRelationalStore::IncreaseConnectionCounter() { @@ -40,7 +48,6 @@ RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) { std::lock_guard lock(connectMutex_); RelationalStoreConnection* connection = new (std::nothrow) SQLiteRelationalStoreConnection(this); - if (connection == nullptr) { errCode = -E_OUT_OF_MEMORY; return nullptr; @@ -50,16 +57,16 @@ RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) return connection; } -static void InitDataBaseOption(const DBProperties &kvDBProp, OpenDbProperties &option) +static void InitDataBaseOption(const RelationalDBProperties &properties, OpenDbProperties &option) { - option.uri = kvDBProp.GetStringProp(KvDBProperties::DATA_DIR, ""); - option.createIfNecessary = kvDBProp.GetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, false); + option.uri = properties.GetStringProp(DBProperties::DATA_DIR, ""); + option.createIfNecessary = properties.GetBoolProp(DBProperties::CREATE_IF_NECESSARY, false); } -int SQLiteRelationalStore::InitStorageEngine(const DBProperties &kvDBProp) +int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &properties) { OpenDbProperties option; - InitDataBaseOption(kvDBProp, option); + InitDataBaseOption(properties, option); StorageEngineAttr poolSize = {1, 1, 0, 16}; // at most 1 write 16 read. int errCode = sqliteStorageEngine_->InitSQLiteStorageEngine(poolSize, option); @@ -69,24 +76,139 @@ int SQLiteRelationalStore::InitStorageEngine(const DBProperties &kvDBProp) return errCode; } -int SQLiteRelationalStore::Open(const DBProperties &properties) +void SQLiteRelationalStore::ReleaseResources() { - sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); - if (sqliteStorageEngine_ == nullptr) { - LOGE("[RelationalStore] Create storage engine failed"); - return -E_OUT_OF_MEMORY; + if (sqliteStorageEngine_ != nullptr) { + sqliteStorageEngine_->ClearEnginePasswd(); + (void)StorageEngineManager::ReleaseStorageEngine(sqliteStorageEngine_); } +} - int errCode = InitStorageEngine(properties); +int SQLiteRelationalStore::CheckDBMode() +{ + int errCode = E_OK; + auto *handle = GetHandle(false, errCode); + if (handle == nullptr) { + return errCode; + } + errCode = handle->CheckDBModeForRelational(); if (errCode != E_OK) { - LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); + LOGE("check relational DB mode failed. %d", errCode); + } + + ReleaseHandle(handle); + return errCode; +} + +int SQLiteRelationalStore::GetSchemaFromMeta() +{ + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + int errCode = storageEngine_->GetMetaData(schemaKey, schemaVal); + if (errCode != E_OK && errCode != -E_NOT_FOUND) { + LOGE("Get relational schema from meta table failed. %d", errCode); return errCode; + } else if (errCode == -E_NOT_FOUND || schemaVal.empty()) { + LOGW("No relational schema info was found."); + return E_OK; } - storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); - syncEngine_ = std::make_shared(storageEngine_); + + std::string schemaStr; + DBCommon::VectorToString(schemaVal, schemaStr); + RelationalSchemaObject schema; + errCode = schema.ParseFromSchemaString(schemaStr); + if (errCode != E_OK) { + LOGE("Parse schema string from mata table failed."); + return errCode; + } + + std::lock_guard lock(schemaMutex_); + properties_.SetSchema(schema); + return E_OK; +} + +int SQLiteRelationalStore::SaveSchemaToMeta() +{ + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + DBCommon::StringToVector(properties_.GetSchema().ToSchemaString(), schemaVal); + int errCode = storageEngine_->PutMetaData(schemaKey, schemaVal); + if (errCode != E_OK) { + LOGE("Save relational schema to meta table failed. %d", errCode); + } + return errCode; +} + +int SQLiteRelationalStore::SaveLogTableVersionToMeta() +{ + LOGD("save log table version to meta table, key: %s, val: %s", LOG_TABLE_VERSION_KEY, LOG_TABLE_VERSION_1); + return E_OK; +} + +int SQLiteRelationalStore::CleanDistributedDeviceTable() +{ return E_OK; } +int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) +{ + std::lock_guard lock(initalMutex_); + if (isInitialized_) { + LOGD("[RelationalStore][Open] relational db was already inited."); + return E_OK; + } + + sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); + if (sqliteStorageEngine_ == nullptr) { + LOGE("[RelationalStore][Open] Create storage engine failed"); + return -E_OUT_OF_MEMORY; + } + + int errCode = E_OK; + do { + errCode = InitStorageEngine(properties); + if (errCode != E_OK) { + LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); + break; + } + + storageEngine_ = new (std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); + if (storageEngine_ == nullptr) { + LOGE("[RelationalStore][Open] Create syncable storage failed"); + errCode = -E_OUT_OF_MEMORY; + break; + } + + errCode = CheckDBMode(); + if (errCode != E_OK) { + break; + } + + properties_ = properties; + errCode = GetSchemaFromMeta(); + if (errCode != E_OK) { + break; + } + + errCode = SaveLogTableVersionToMeta(); + if (errCode != E_OK) { + break; + } + + errCode = CleanDistributedDeviceTable(); + if (errCode != E_OK) { + break; + } + + syncEngine_ = std::make_unique(storageEngine_); + isInitialized_ = true; + return E_OK; + } while (false); + + ReleaseResources(); + return errCode; +} + void SQLiteRelationalStore::OnClose(const std::function ¬ifier) { AutoLock lockGuard(this); @@ -179,5 +301,49 @@ void SQLiteRelationalStore::WakeUpSyncer() { syncEngine_->WakeUpSyncer(); } + + +int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) +{ + int errCode = E_OK; + std::lock_guard lock(schemaMutex_); + auto schema = properties_.GetSchema(); + if (schema.GetTable(tableName).GetTableName() == tableName) { + LOGW("distributed table was already created."); + return E_OK; + } + + if (schema.GetTables().size() >= DBConstant::MAX_DISTRIBUTED_TABLE_COUNT) { + LOGW("The number of distributed tables is exceeds limit."); + return -E_MAX_LIMITS; + } + + LOGD("Create distributed table."); + auto *handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + + TableInfo table; + errCode = handle->CreateDistributedTable(tableName, table); + if (errCode != E_OK) { + LOGE("create distributed table failed. %d", errCode); + (void)handle->Rollback(); + ReleaseHandle(handle); + return errCode; + } + schema.AddRelationalTable(table); + properties_.SetSchema(schema); + (void)handle->Commit(); + + ReleaseHandle(handle); + return SaveSchemaToMeta(); +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index f41dd0839819f9e02d4a1933561241f3121d9789..ebb3fb2b5949d83e2338887311b2f409eccfcd34 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -17,7 +17,7 @@ #ifdef RELATIONAL_STORE #include -#include +#include #include #include "irelational_store.h" @@ -33,7 +33,7 @@ public: ~SQLiteRelationalStore() override; RelationalStoreConnection *GetDBConnection(int &errCode) override; - int Open(const DBProperties &properties) override; + int Open(const RelationalDBProperties &properties) override; void OnClose(const std::function ¬ifier); SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode) const; @@ -43,28 +43,46 @@ public: void ReleaseDBConnection(RelationalStoreConnection *connection); - void WakeUpSyncer(); + void WakeUpSyncer() override; // for test mock const RelationalSyncAbleStorage *GetStorageEngine() { return storageEngine_; } + + int CreateDistributedTable(const std::string &tableName); private: + void ReleaseResources(); + // 1 store 1 connection void DecreaseConnectionCounter(); + int CheckDBMode(); + int GetSchemaFromMeta(); + int SaveSchemaToMeta(); + + int SaveLogTableVersionToMeta(); + + int CleanDistributedDeviceTable(); // use for sync Interactive - std::shared_ptr syncEngine_ = nullptr; // For storage operate sync function + std::unique_ptr syncEngine_ = nullptr; // For storage operate sync function // use ref obj same as kv RelationalSyncAbleStorage *storageEngine_ = nullptr; // For storage operate data SQLiteSingleRelationalStorageEngine *sqliteStorageEngine_ = nullptr; void IncreaseConnectionCounter(); - int InitStorageEngine(const DBProperties &kvDBProp); + int InitStorageEngine(const RelationalDBProperties &kvDBProp); std::mutex connectMutex_; std::atomic connectionCount_ = 0; std::vector> closeNotifiers_; + + RelationalDBProperties properties_; + + mutable std::mutex initalMutex_; + bool isInitialized_ = false; + + mutable std::mutex schemaMutex_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index d322575d709847e339e745318ea1d06e5b383aff..de529540bcc2f106bd9f018634fbc82fa80bfcb5 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -58,7 +58,7 @@ SQLiteSingleVerRelationalStorageExecutor *SQLiteRelationalStoreConnection::GetEx auto *store = GetDB(); if (store == nullptr) { errCode = -E_NOT_INIT; - LOGE("[SingleVerConnection] store is null, get executor failed! errCode = [%d]", errCode); + LOGE("[RelationalConnection] store is null, get executor failed! errCode = [%d]", errCode); return nullptr; } @@ -93,7 +93,7 @@ int SQLiteRelationalStoreConnection::StartTransaction() return errCode; } - LOGD("[SingleVerConnection] Start transaction finish."); + LOGD("[RelationalConnection] Start transaction finish."); writeHandle_ = handle; transactingFlag_.store(true); return E_OK; @@ -135,20 +135,17 @@ int SQLiteRelationalStoreConnection::RollBack() return errCode; } -int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) +int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &tableName) { - int errCode = StartTransaction(); - if (errCode != E_OK && errCode != E_TRANSACT_STATE) { - return errCode; + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[RelationalConnection] store is null, get DB failed!"); + return -E_INVALID_CONNECTION; } - errCode = writeHandle_->CreateDistributedTable(tableName, option); - if (errCode == E_OK) { - errCode = Commit(); - } else { - int innerCode = RollBack(); - errCode = (innerCode != E_OK) ? innerCode : errCode; + int errCode = store->CreateDistributedTable(tableName); + if (errCode != E_OK) { + LOGE("[RelationalConnection] crete distributed table failed. %d", errCode); } return errCode; } @@ -167,7 +164,7 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) { auto *store = GetDB(); if (store == nullptr) { - LOGE("[SingleVerConnection] store is null, get executor failed!"); + LOGE("[RelationalConnection] store is null, get executor failed!"); return -E_INVALID_CONNECTION; } @@ -190,5 +187,10 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) } return E_OK; } + +int SQLiteRelationalStoreConnection::RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) +{ + return E_OK; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 42184112fc0917c0a31fdbf19fc1e440db58044c..552447e76e6ab4e6aae8ae70aa84d35df375cfe4 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -37,12 +37,12 @@ public: int TriggerAutoSync() override; int SyncToDevice(SyncInfo &info) override; std::string GetIdentifier() override; - int CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) override; + int CreateDistributedTable(const std::string &tableName) override; + int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) override; protected: - int Pragma(int cmd, void *parameter); + int Pragma(int cmd, void *parameter) override; private: SQLiteSingleVerRelationalStorageExecutor *GetExecutor(bool isWrite, int &errCode) const; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index ab12fd5d85fe01e9b000b2c3c335c35299da09ba..6ef4b916eda3dd8d12d30783bbbaeff0ca1ec604 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -52,12 +52,12 @@ int SQLiteSingleRelationalStorageEngine::RegisterFunction(sqlite3 *db) const int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, StorageExecutor *&handle) { sqlite3 *db = nullptr; - int errCode = SQLiteUtils::OpenDatabase(option_, db); + int errCode = SQLiteUtils::OpenDatabase(option_, db, false); if (errCode != E_OK) { return errCode; } do { - errCode = Upgrade(db); + errCode = Upgrade(db); // cerate meta_data table. if (errCode != E_OK) { break; } @@ -66,6 +66,7 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage if (errCode != E_OK) { break; } + handle = NewSQLiteStorageExecutor(db, isWrite, false); if (handle == nullptr) { LOGE("[Relational] New SQLiteStorageExecutor[%d] for the pool failed.", isWrite); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h index 28ea03efbf03f16fd3f4e4215b6bd46486572099..7bb37460b82199c06a3b41a95629f51e5d2d2b78 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h @@ -35,14 +35,8 @@ protected: int CreateNewExecutor(bool isWrite, StorageExecutor *&handle) override; private: - // For executor. - - // For engine. - - // For db. int RegisterFunction(sqlite3 *db) const; }; } // namespace DistributedDB - #endif #endif // SQLITE_RELATIONAL_ENGINE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp index 2053fb76b9990f533e89313d0e03b4bcbf4e407b..212b13f301e0b457e2d708a93dabca385c0368a6 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp @@ -83,6 +83,38 @@ std::string FieldValue2String(const FieldValue &val, QueryValueType type) return ""; } } + +std::string GetSelectAndFromClauseForRDB(const std::string &tableName) +{ + return "SELECT b.data_key," + "b.device," + "b.ori_device," + "b.timestamp as " + DBConstant::TIMESTAMP_ALIAS + "," + "b.wtimestamp," + "b.flag," + "b.hash_key," + "a.* " + "FROM " + tableName + " AS a INNER JOIN " + DBConstant::RELATIONAL_PREFIX + tableName + "_log AS b " + "ON a.rowid=b.data_key "; +} + +std::string GetTimeRangeClauseForRDB() +{ + return " AND (" + DBConstant::TIMESTAMP_ALIAS + ">=? AND " + DBConstant::TIMESTAMP_ALIAS + "=? AND " + DBConstant::TIMESTAMP_ALIAS + "=? AND naturalbase_rdb_timestamp=? AND naturalbase_rdb_timestamp>> mulDevTableTimeRange_; - std::map>> mulDevTableDeletedTimeRange_; }; } // namespace DistributedDB #endif // SQLITE_SINGLE_VER_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp index 80a98f372fd1b178679779820e63017a6b98af8c..1110153a52df5a5f91e8567a9e4702eb7ee132ae 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp @@ -44,10 +44,6 @@ namespace DistributedDB { namespace { constexpr int DEF_LIFE_CYCLE_TIME = 60000; // 60s constexpr int WAIT_DELEGATE_CALLBACK_TIME = 100; - // In querySync, when getting query data finished, - // if the block size reach the half of max block size, will get deleted data next; - // if the block size not reach the half of max block size, will not get deleted data. - constexpr float QUERY_SYNC_THRESHOLD = 0.50; const std::string CREATE_DB_TIME = "createDBTime"; @@ -161,7 +157,7 @@ namespace { bool reachThreshold = false; for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { blockSize += SQLiteSingleVerStorageExecutor::GetDataItemSerialSize(dataItems[i], appendLen); - reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); + reachThreshold = (blockSize >= dataSizeInfo.blockSize * DBConstant::QUERY_SYNC_THRESHOLD); } return !reachThreshold; } @@ -178,7 +174,8 @@ SQLiteSingleVerNaturalStore::SQLiteSingleVerNaturalStore() lifeTimerId_(0), autoLifeTime_(DEF_LIFE_CYCLE_TIME), createDBTime_(0), - dataInterceptor_(nullptr) + dataInterceptor_(nullptr), + maxLogSize_(DBConstant::MAX_LOG_SIZE_DEFAULT) {} SQLiteSingleVerNaturalStore::~SQLiteSingleVerNaturalStore() @@ -372,6 +369,9 @@ int SQLiteSingleVerNaturalStore::ClearIncompleteDatabase(const KvDBProperties &k int SQLiteSingleVerNaturalStore::CheckDatabaseRecovery(const KvDBProperties &kvDBProp) { + if (kvDBProp.GetBoolProp(KvDBProperties::MEMORY_MODE, false)) { // memory status not need recovery + return E_OK; + } std::unique_ptr operation = std::make_unique(this, nullptr); (void)operation->ClearExportedTempFiles(kvDBProp); int errCode = operation->RekeyRecover(kvDBProp); @@ -942,8 +942,21 @@ int SQLiteSingleVerNaturalStore::RemoveDeviceData(const std::string &deviceName, if (!isInSync && !CheckWritePermission()) { return -E_NOT_PERMIT; } + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetHandle(true, errCode); + if (handle == nullptr) { + LOGE("[SingleVerNStore] RemoveDeviceData get handle failed:%d", errCode); + return errCode; + } + uint64_t logFileSize = handle->GetLogFileSize(); + ReleaseHandle(handle); + if (logFileSize > GetMaxLogSize()) { + LOGW("[SingleVerNStore] RmDevData log size[%llu] over the limit", logFileSize); + return -E_LOG_OVER_LIMITS; + } + // Call the syncer module to erase the water mark. - int errCode = EraseDeviceWaterMark(deviceName, true); + errCode = EraseDeviceWaterMark(deviceName, true); if (errCode != E_OK) { LOGE("[SingleVerNStore] erase water mark failed:%d", errCode); return errCode; @@ -2218,6 +2231,17 @@ int SQLiteSingleVerNaturalStore::RemoveSubscribe(const std::string &subscribeId) return RemoveSubscribe(std::vector {subscribeId}); } +int SQLiteSingleVerNaturalStore::SetMaxLogSize(uint64_t limit) +{ + LOGI("Set the max log size to %llu", limit); + maxLogSize_.store(limit); + return E_OK; +} +uint64_t SQLiteSingleVerNaturalStore::GetMaxLogSize() const +{ + return maxLogSize_.load(); +} + int SQLiteSingleVerNaturalStore::RemoveAllSubscribe() { int errCode = E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h index bc4b5795f0af2173f905232d40c67aa9458a1b82..8e02212b2a4d0f4d8ee670ba3067574e6fe4eada 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h @@ -183,6 +183,10 @@ public: int RemoveSubscribe(const std::vector &subscribeIds) override; + int SetMaxLogSize(uint64_t limit); + + uint64_t GetMaxLogSize() const; + private: int CheckDatabaseRecovery(const KvDBProperties &kvDBProp); @@ -279,6 +283,7 @@ private: mutable std::shared_mutex dataInterceptorMutex_; PushDataInterceptor dataInterceptor_; + std::atomic maxLogSize_; }; } #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp index fc54546f8160acbf6b8724f19370f345557465a4..40a692b647d97de9d3cb1132928b073c9473c939 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp @@ -397,6 +397,10 @@ int SQLiteSingleVerNaturalStoreConnection::Pragma(int cmd, void *parameter) return PragmaResultSetCacheMaxSize(parameter); case PRAGMA_TRIGGER_TO_MIGRATE_DATA: return PragmaTriggerToMigrateData(*static_cast(parameter)); + case PRAGMA_SET_MAX_LOG_SIZE: + return PragmaSetMaxLogSize(static_cast(parameter)); + case PRAGMA_EXEC_CHECKPOINT: + return ForceCheckPoint(); default: // Call Pragma() of super class. errCode = SyncAbleKvDBConnection::Pragma(cmd, parameter); @@ -724,6 +728,36 @@ int SQLiteSingleVerNaturalStoreConnection::CheckIntegrity() const return errCode; } +int SQLiteSingleVerNaturalStoreConnection::PragmaSetMaxLogSize(uint64_t *limit) +{ + if (limit == nullptr) { + return -E_INVALID_ARGS; + } + SQLiteSingleVerNaturalStore *naturalStore = GetDB(); + if (naturalStore == nullptr) { + LOGE("[SingleVerConnection] db is nullptr for max log limit set."); + return -E_INVALID_DB; + } + if (*limit > DBConstant::MAX_LOG_SIZE_HIGH || *limit < DBConstant::MAX_LOG_SIZE_LOW) { + return -E_INVALID_ARGS; + } + return naturalStore->SetMaxLogSize(*limit); +} + +int SQLiteSingleVerNaturalStoreConnection::ForceCheckPoint() const +{ + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetExecutor(true, errCode); + if (handle == nullptr) { + LOGW("Failed to get the executor for the checkpoint."); + return errCode; + } + + errCode = handle->ForceCheckPoint(); + ReleaseExecutor(handle); + return errCode; +} + int SQLiteSingleVerNaturalStoreConnection::CheckMonoStatus(OperatePerm perm) { // 1. Get the connection number @@ -1218,6 +1252,11 @@ int SQLiteSingleVerNaturalStoreConnection::StartTransactionInCacheMode() if (handle == nullptr) { return errCode; } + if (CheckLogOverLimit(handle)) { + LOGW("Over the log limit"); + ReleaseExecutor(handle); + return -E_LOG_OVER_LIMITS; + } errCode = handle->StartTransaction(TransactType::DEFERRED); if (errCode != E_OK) { ReleaseExecutor(handle); @@ -1237,6 +1276,11 @@ int SQLiteSingleVerNaturalStoreConnection::StartTransactionNormally() if (handle == nullptr) { return errCode; } + if (CheckLogOverLimit(handle)) { + LOGW("Over the log limit"); + ReleaseExecutor(handle); + return -E_LOG_OVER_LIMITS; + } if (committedData_ == nullptr) { committedData_ = new (std::nothrow) SingleVerNaturalStoreCommitNotifyData; @@ -1706,5 +1750,19 @@ int SQLiteSingleVerNaturalStoreConnection::CheckAmendValueContentForLocalProcedu return naturalStore->CheckValueAndAmendIfNeed(ValueSource::FROM_LOCAL, oriValue, amendValue, useAmendValue); } +bool SQLiteSingleVerNaturalStoreConnection::CheckLogOverLimit(SQLiteSingleVerStorageExecutor *executor) const +{ + SQLiteSingleVerNaturalStore *naturalStore = GetDB(); + if (naturalStore == nullptr || executor == nullptr) { // Not Likely + return false; + } + uint64_t logFileSize = executor->GetLogFileSize(); + bool result = logFileSize > naturalStore->GetMaxLogSize(); + if (result) { + LOGW("Log size[%llu] over the limit", logFileSize); + } + return result; +} + DEFINE_OBJECT_TAG_FACILITIES(SQLiteSingleVerNaturalStoreConnection) } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h index 1bd8f40081d790006a7314f9005e7be6a10eaa7c..82a5fb6e4523c767740382f730b8a8ea3ad4a017 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h @@ -196,6 +196,11 @@ private: int CheckReadDataControlled() const; bool IsFileAccessControlled() const; + int PragmaSetMaxLogSize(uint64_t *limit); + int ForceCheckPoint() const; + + bool CheckLogOverLimit(SQLiteSingleVerStorageExecutor *executor) const; + DECLARE_OBJECT_TAG(SQLiteSingleVerNaturalStoreConnection); // ResultSet Related Info diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6e61927335272dcbc1ca4cdab72d8980b36719be --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "sqlite_single_ver_relational_continue_token.h" +#include "sqlite_utils.h" + +namespace DistributedDB { +SQLiteSingleVerRelationalContinueToken::SQLiteSingleVerRelationalContinueToken( + const SyncTimeRange &timeRange, const QueryObject &object) + : isGettingDeletedData_(false), queryObj_(object), tableName_(object.GetTableName()), timeRange_(timeRange) +{} + +bool SQLiteSingleVerRelationalContinueToken::CheckValid() const +{ + bool isValid = (magicBegin_ == MAGIC_BEGIN && magicEnd_ == MAGIC_END); + if (!isValid) { + LOGE("Invalid continue token."); + } + return isValid; +} + +int SQLiteSingleVerRelationalContinueToken::GetStatement(sqlite3 *db, sqlite3_stmt *&stmt, bool &isGettingDeletedData) +{ + isGettingDeletedData = isGettingDeletedData_; + if (isGettingDeletedData) { + return GetDeletedDataStmt(db, stmt); + } + return GetQuerySyncStatement(db, stmt); +} + +void SQLiteSingleVerRelationalContinueToken::SetNextBeginTime(const DataItem &theLastItem) +{ + TimeStamp nextBeginTime = theLastItem.timeStamp + 1; + if (nextBeginTime > INT64_MAX) { + nextBeginTime = INT64_MAX; + } + if (!isGettingDeletedData_) { + timeRange_.beginTime = nextBeginTime; + return; + } + if ((theLastItem.flag & DataItem::DELETE_FLAG) != 0) { // The last one could be non-deleted. + timeRange_.deleteBeginTime = nextBeginTime; + } +} + +void SQLiteSingleVerRelationalContinueToken::FinishGetData() +{ + if (isGettingDeletedData_) { + timeRange_.deleteEndTime = 0; + return; + } + isGettingDeletedData_ = true; + timeRange_.endTime = 0; + return; +} + +bool SQLiteSingleVerRelationalContinueToken::IsGetAllDataFinished() const +{ + return timeRange_.beginTime >= timeRange_.endTime && timeRange_.deleteBeginTime >= timeRange_.deleteEndTime; +} + +int SQLiteSingleVerRelationalContinueToken::GetQuerySyncStatement(sqlite3 *db, sqlite3_stmt *&stmt) +{ + int errCode = E_OK; + SqliteQueryHelper helper = queryObj_.GetQueryHelper(errCode); + if (errCode != E_OK) { + return errCode; + } + return helper.GetRelationalQueryStatement(db, timeRange_.beginTime, timeRange_.endTime, stmt); +} + +int SQLiteSingleVerRelationalContinueToken::GetDeletedDataStmt(sqlite3 *db, sqlite3_stmt *&stmt) const +{ + // get stmt + const std::string sql = GetDeletedDataSQL(); + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + goto ERROR; + } + + // bind stmt + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 1, timeRange_.deleteBeginTime); // 1 means begin time + if (errCode != E_OK) { + goto ERROR; + } + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 2, timeRange_.deleteEndTime); // 2 means end time + if (errCode != E_OK) { + goto ERROR; + } + return errCode; + +ERROR: + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} + +const QueryObject &SQLiteSingleVerRelationalContinueToken::GetQuery() const +{ + return queryObj_; +} + +std::string SQLiteSingleVerRelationalContinueToken::GetDeletedDataSQL() const +{ + std::string tableName = DBConstant::RELATIONAL_PREFIX + tableName_ + "_log"; + return "SELECT * FROM " + tableName + + " WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03 = 0x03) ORDER BY timestamp ASC;"; +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h old mode 100755 new mode 100644 similarity index 31% rename from services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h rename to services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h index 6d77f0ccfc7a011fa5aea8f8d811cb7040223d05..912aa4c960addb2084a038d0d9b51241e709ac82 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h @@ -12,34 +12,44 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#ifndef SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H +#define SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H +#ifdef RELATIONAL_STORE +#include +#include -#ifndef SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H -#define SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H - -#include "sliding_window_receiver.h" -#include "sliding_window_sender.h" +#include "db_types.h" +#include "query_object.h" namespace DistributedDB { -class SingleVerDataSyncWithSlidingWindow { +class SQLiteSingleVerRelationalContinueToken { public: - SingleVerDataSyncWithSlidingWindow() = default; - ~SingleVerDataSyncWithSlidingWindow() = default; - DISABLE_COPY_ASSIGN_MOVE(SingleVerDataSyncWithSlidingWindow); + SQLiteSingleVerRelationalContinueToken(const SyncTimeRange &timeRange, const QueryObject &queryObject); + ~SQLiteSingleVerRelationalContinueToken() = default; - int SenderStart(int32_t mode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int PreHandleSenderAckRecv(const Message *message); - int SenderAckRecv(const Message *message); - void SenderClear(); - void SetSenderErr(bool isErr); + // Check the magic number at the beginning and end of the SQLiteSingleVerRelationalContinueToken. + bool CheckValid() const; + // The statement should be release by the caller. + int GetStatement(sqlite3 *db, sqlite3_stmt *&stmt, bool &isGettingDeletedData); + void SetNextBeginTime(const DataItem &theLastItem); + void FinishGetData(); + bool IsGetAllDataFinished() const; + const QueryObject &GetQuery() const; - int ReceiverInit(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int SenderInit(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - void ReceiverClear(); - int Receive(Message *inMsg); private: - SlidingWindowSender sender_; - SlidingWindowReceiver receiver_; + std::string GetDeletedDataSQL() const; + int GetQuerySyncStatement(sqlite3 *db, sqlite3_stmt *&stmt); + int GetDeletedDataStmt(sqlite3 *db, sqlite3_stmt *&stmt) const; + + static const unsigned int MAGIC_BEGIN = 0x600D0AC7; // for token guard + static const unsigned int MAGIC_END = 0x0AC7600D; // for token guard + unsigned int magicBegin_ = MAGIC_BEGIN; + int isGettingDeletedData_ = false; + QueryObject queryObj_; + const std::string &tableName_; + SyncTimeRange timeRange_; + unsigned int magicEnd_ = MAGIC_END; }; } // namespace DistributedDB -#endif // SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H - +#endif // RELATIONAL_STORE +#endif // SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp similarity index 67% rename from services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp rename to services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 167adfdb290ac6848db05b7e2b227d4a85c24812..fd765798715232a063e80d87fb07c0752dcab9d2 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -19,23 +19,39 @@ namespace DistributedDB { SQLiteSingleVerRelationalStorageExecutor::SQLiteSingleVerRelationalStorageExecutor(sqlite3 *dbHandle, bool writable) - : SQLiteStorageExecutor(dbHandle, writable, false) {}; + : SQLiteStorageExecutor(dbHandle, writable, false) +{} -int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) +int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, TableInfo &table) { if (dbHandle_ == nullptr) { - LOGE("Begin transaction failed, dbHandle is null."); + LOGE("[CreateDistributedTable] Begin transaction failed, dbHandle is null."); return -E_INVALID_DB; } - TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode != E_OK) { - LOGE("[CreateDistributedTable] analysis table schema failed"); + LOGE("[CreateDistributedTable] analysis table schema failed. %d", errCode); return errCode; } + if (table.GetCreateTableSql().find("WITHOUT ROWID") != std::string::npos) { + LOGE("[CreateDistributedTable] Not support create distributed table without rowid."); + return -E_NOT_SUPPORT; + } + + bool isTableEmpty = false; + errCode = SQLiteUtils::CheckTableEmpty(dbHandle_, tableName, isTableEmpty); + if (errCode != E_OK) { + LOGE("[CreateDistributedTable] Check table [%s] is empty failed. %d", tableName.c_str(), errCode); + return errCode; + } + + if (!isTableEmpty) { // create distributed table should on an empty table + LOGE("[CreateDistributedTable] Create distributed table should on an empty table."); + return -E_NOT_SUPPORT; + } + // create log table errCode = SQLiteUtils::CreateRelationalLogTable(dbHandle_, tableName); if (errCode != E_OK) { @@ -86,7 +102,7 @@ int SQLiteSingleVerRelationalStorageExecutor::Rollback() return errCode; } -int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) +int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(const QueryObject &query) { int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); if (errCode != E_OK) { @@ -95,168 +111,135 @@ int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) return errCode; } -// binding index just for the get sync data sql -static const int BIND_BEGIN_STAMP_INDEX = 1; -static const int BIND_END_STAMP_INDEX = 2; - -int SQLiteSingleVerRelationalStorageExecutor::PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, - sqlite3_stmt *&statement, bool getDeletedData) const +static int GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) { - if (dbHandle_ == nullptr) { - return -E_INVALID_DB; - } - - const std::string SELECT_SYNC_DELETED_ENTRIES_SQL = - "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + - "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x03) ORDER BY timestamp ASC;"; - const std::string SELECT_SYNC_ENTRIES_SQL = - "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + - "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x02=0x02) ORDER BY timestamp ASC;"; - - const std::string sql = (getDeletedData ? SELECT_SYNC_DELETED_ENTRIES_SQL : SELECT_SYNC_ENTRIES_SQL); - int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); - if (errCode != E_OK) { - LOGE("Prepare the sync entries statement error:%d", errCode); - return errCode; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_BEGIN_STAMP_INDEX, begin); - if (errCode != E_OK) { - goto ERROR; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_END_STAMP_INDEX, end); - if (errCode != E_OK) { - goto ERROR; - } - -ERROR: - if (errCode != E_OK) { - LOGE("Bind the timestamp for getting sync data error:%d", errCode); - SQLiteUtils::ResetStatement(statement, true, errCode); - } - - return CheckCorruptedStatus(errCode); -} - -static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) -{ - switch (type) { - case StorageType::STORAGE_TYPE_BOOL: { - value = static_cast(sqlite3_column_int64(statement, cid)); + int errCode = E_OK; + int storageType = sqlite3_column_type(statement, cid); + switch (storageType) { + case SQLITE_INTEGER: { + value = static_cast(sqlite3_column_int64(statement, cid)); break; } - - case StorageType::STORAGE_TYPE_INTEGER: { - int64_t v = static_cast(sqlite3_column_int64(statement, cid)); - value = v; + case SQLITE_FLOAT: { + value = sqlite3_column_double(statement, cid); break; } - - case StorageType::STORAGE_TYPE_REAL: { - value = sqlite3_column_double(statement, cid); + case SQLITE_BLOB: { + std::vector blobValue; + errCode = SQLiteUtils::GetColumnBlobValue(statement, cid, blobValue); + if (errCode != E_OK) { + return errCode; + } + auto blob = new (std::nothrow) Blob; + if (blob == nullptr) { + return -E_OUT_OF_MEMORY; + } + blob->WriteBlob(blobValue.data(), static_cast(blobValue.size())); + errCode = value.Set(blob); break; } - - case StorageType::STORAGE_TYPE_TEXT: { - const char *colValue = reinterpret_cast(sqlite3_column_text(statement, cid)); + case SQLITE_NULL: { + break; + } + case SQLITE3_TEXT: { + const char *colValue = reinterpret_cast(sqlite3_column_text(statement, cid)); if (colValue == nullptr) { - value = std::string(); + value.ResetValue(); } else { value = std::string(colValue); + if (value.GetType() == StorageType::STORAGE_TYPE_NULL) { + errCode = -E_OUT_OF_MEMORY; + } } break; } - - case StorageType::STORAGE_TYPE_BLOB: { - std::vector blobValue; - (void)SQLiteUtils::GetColumnBlobValue(statement, cid, blobValue); - Blob blob; - blob.WriteBlob(blobValue.data(), static_cast(blobValue.size())); - value = blob; + default: { break; } - - default: - return; } + return errCode; } -static void BindDataValueByType(sqlite3_stmt *statement, - const std::optional &value, StorageType type, int cid) +static int BindDataValueByType(sqlite3_stmt *statement, const std::optional &data, int cid) { - if (value == std::nullopt) { - sqlite3_bind_null(statement, cid); - return; + if (!data.has_value()) { // For the column that added after enable distributed. + return E_OK; } + int errCode = E_OK; + StorageType type = data.value().GetType(); switch (type) { case StorageType::STORAGE_TYPE_BOOL: { bool boolData = false; - value.value().GetBool(boolData); - sqlite3_bind_int(statement, cid, boolData); + (void)data.value().GetBool(boolData); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_int(statement, cid, boolData)); break; } case StorageType::STORAGE_TYPE_INTEGER: { int64_t intData = 0; - value.value().GetInt64(intData); - sqlite3_bind_int64(statement, cid, intData); + (void)data.value().GetInt64(intData); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_int64(statement, cid, intData)); break; } case StorageType::STORAGE_TYPE_REAL: { double doubleData = 0; - value.value().GetDouble(doubleData); - sqlite3_bind_double(statement, cid, doubleData); + (void)data.value().GetDouble(doubleData); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_double(statement, cid, doubleData)); break; } case StorageType::STORAGE_TYPE_TEXT: { std::string strData; - value.value().GetText(strData); - SQLiteUtils::BindTextToStatement(statement, cid, strData); + (void)data.value().GetText(strData); + errCode = SQLiteUtils::BindTextToStatement(statement, cid, strData); break; } case StorageType::STORAGE_TYPE_BLOB: { Blob blob; - value.value().GetBlob(blob); + (void)data.value().GetBlob(blob); std::vector blobData(blob.GetData(), blob.GetData() + blob.GetSize()); - SQLiteUtils::BindBlobToStatement(statement, cid, blobData, true); + errCode = SQLiteUtils::BindBlobToStatement(statement, cid, blobData, true); + break; + } + + case StorageType::STORAGE_TYPE_NULL: { + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_null(statement, cid)); break; } default: - return; + break; } + return errCode; } static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) { - logInfo.dataKey = sqlite3_column_int(logStatement, 0); + logInfo.dataKey = sqlite3_column_int(logStatement, 0); // 0 means dataKey index std::vector dev; - int errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 1, dev); + int errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 1, dev); // 1 means dev index if (errCode != E_OK) { return errCode; } logInfo.device = std::string(dev.begin(), dev.end()); std::vector oriDev; - errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 2, oriDev); + errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 2, oriDev); // 2 means ori_dev index if (errCode != E_OK) { return errCode; } logInfo.originDev = std::string(oriDev.begin(), oriDev.end()); - - logInfo.timestamp = static_cast(sqlite3_column_int64(logStatement, 3)); - logInfo.wTimeStamp = static_cast(sqlite3_column_int64(logStatement, 4)); - logInfo.flag = static_cast(sqlite3_column_int64(logStatement, 5)); + logInfo.timestamp = static_cast(sqlite3_column_int64(logStatement, 3)); // 3 means timestamp index + logInfo.wTimeStamp = static_cast(sqlite3_column_int64(logStatement, 4)); // 4 means w_timestamp index + logInfo.flag = static_cast(sqlite3_column_int64(logStatement, 5)); // 5 means flag index logInfo.flag &= (~DataItem::LOCAL_FLAG); std::vector hashKey; - errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 6, hashKey); + errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 6, hashKey); // 6 means hashKey index if (errCode != E_OK) { return errCode; } @@ -265,68 +248,6 @@ static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) return errCode; } -static sqlite3_stmt *GetDataStmtByPK(sqlite3 *db, const std::string &tableName, int rowId) -{ - std::string sql = "select * from " + tableName + " where rowId=?;"; - sqlite3_stmt *statement = nullptr; - int errCode = SQLiteUtils::GetStatement(db, sql, statement); - if (errCode != E_OK) { - LOGE("[data by rowid statement] Get statement fail!"); - return nullptr; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, 1, rowId); - if (errCode != E_OK) { - SQLiteUtils::ResetStatement(statement, true, errCode); - return nullptr; - } - - return statement; -} - - -int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *logStatement, DataItem &dataItem) const -{ - std::map colInfos = table_.GetFields(); - RowDataWithLog data; - - int errCode = GetLogData(logStatement, data.logInfo); - if (errCode != E_OK) { - LOGE("relational data value transfer to kv fail"); - return errCode; - } - - // dataKey is rowid by pk - // only get local device data - sqlite3_stmt *stmt = GetDataStmtByPK(dbHandle_, table_.GetTableName(), data.logInfo.dataKey); - - errCode = SQLiteUtils::StepWithRetry(stmt, false); - if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW) && errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGE("[SQLiteSingleVerRelationalStorageExecutor] Step fail errCode:%d", errCode); - return errCode; - } - - std::vector fieldInfos; - for (const auto &col: colInfos) { - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - break; - } - DataValue value; - GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId()); - - fieldInfos.push_back(col.second); - data.rowData.push_back(value); - } - - SQLiteUtils::ResetStatement(stmt, true, errCode); - - errCode = DataTransformer::SerializeDataItem(data, fieldInfos, dataItem); - if (errCode != E_OK) { - LOGE("relational data value transfer to kv fail"); - } - return errCode; -} - static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) { // timestamp and local flag: 3 * uint64_t, version(uint32_t), key, value, origin dev and the padding size. @@ -338,57 +259,10 @@ static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) return dataSize; } -int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataItems(std::vector &dataItems, - sqlite3_stmt *logStatement, size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const -{ - int errCode; - size_t dataTotalSize = 0; - do { - DataItem item; - errCode = SQLiteUtils::StepWithRetry(logStatement, isMemDb_); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - errCode = GetDataItemForSync(logStatement, item); - } else { - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); - errCode = -E_FINISHED; - } else { - LOGE("Get sync data error:%d", errCode); - } - break; - } - - // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. - dataTotalSize += GetDataItemSerialSize(item, appendLength); - if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || - dataItems.size() >= dataSizeInfo.packetSize) { - errCode = -E_UNFINISHED; - break; - } else { - dataItems.push_back(std::move(item)); - } - } while (true); - return errCode; -} - -int SQLiteSingleVerRelationalStorageExecutor::GetDeletedSyncDataByTimestamp(std::vector &dataItems, - size_t appendLength, TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const -{ - sqlite3_stmt *logStatement = nullptr; - int errCode = PrepareForSyncDataByTime(begin, end, logStatement, true); - if (errCode != E_OK) { - return errCode; - } - - errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); - SQLiteUtils::ResetStatement(logStatement, true, errCode); - return CheckCorruptedStatus(errCode); -} - -static const std::string SELECT_META_VALUE_SQL = - "SELECT value FROM distributeddatamgr_aux_metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &value) const { + static const std::string SELECT_META_VALUE_SQL = "SELECT value FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_META_VALUE_SQL, statement); if (errCode != E_OK) { @@ -414,22 +288,23 @@ int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &v return errCode; } -static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO distributeddatamgr_aux_metadata VALUES(?,?);"; int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const { + static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + + "metadata VALUES(?,?);"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, INSERT_META_SQL, statement); if (errCode != E_OK) { goto ERROR; } - errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // BIND_KV_KEY_INDEX + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // 1 means key index if (errCode != E_OK) { LOGE("[SingleVerExe][BindPutKv]Bind key error:%d", errCode); goto ERROR; } - errCode = SQLiteUtils::BindBlobToStatement(statement, 2, value, true); // BIND_KV_VAL_INDEX + errCode = SQLiteUtils::BindBlobToStatement(statement, 2, value, true); // 2 means value index if (errCode != E_OK) { LOGE("[SingleVerExe][BindPutKv]Bind value error:%d", errCode); goto ERROR; @@ -443,9 +318,10 @@ ERROR: return errCode; } -static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM distributeddatamgr_aux_metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const { + static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_SQL, statement); if (errCode != E_OK) { @@ -469,10 +345,10 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector=? AND key<=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const { + static const std::string REMOVE_META_VALUE_BY_KEY_PREFIX_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key>=? AND key<=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_BY_KEY_PREFIX_SQL, statement); if (errCode != E_OK) { @@ -517,10 +393,9 @@ static int GetAllKeys(sqlite3_stmt *statement, std::vector &keys) return errCode; } - -static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM distributeddatamgr_aux_metadata;"; int SQLiteSingleVerRelationalStorageExecutor::GetAllMetaKeys(std::vector &keys) const { + static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM " + DBConstant::RELATIONAL_PREFIX + "metadata;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_ALL_META_KEYS, statement); if (errCode != E_OK) { @@ -536,7 +411,7 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj const std::string &deviceName, sqlite3_stmt *&logStmt) const { std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_log"; + const std::string tableName = DBConstant::RELATIONAL_PREFIX + object.GetTableName() + "_log"; std::string dataFormat = "?, '" + deviceName + "', ?, ?, ?, ?, ?"; std::string sql = "INSERT OR REPLACE INTO " + tableName + @@ -552,15 +427,18 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const { - // distributeddatamgr_aux_userTableName_deviceHash + // naturalbase_rdb_aux_userTableName_deviceHash // tableName std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_" + + const std::string tableName = DBConstant::RELATIONAL_PREFIX + object.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode == -E_NOT_FOUND) { errCode = SQLiteUtils::CreateSameStuTable(dbHandle_, object.GetTableName(), tableName, false); + if (errCode == E_OK) { + errCode = SQLiteUtils::CloneIndexes(dbHandle_, object.GetTableName(), tableName); + } } if (errCode != E_OK) { @@ -587,13 +465,14 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryOb return errCode; } -int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statement, - const DataItem &dataItem, TimeStamp &maxTimestamp) +int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statement, const DataItem &dataItem, + TimeStamp &maxTimestamp) { Key hashKey; (void)DBCommon::CalcValueHash(dataItem.key, hashKey); std::string hash = std::string(hashKey.begin(), hashKey.end()); - std::string sql = "select * from distributeddatamgr_aux_" + table_.GetTableName() + "_log where hash_key = ?;"; + std::string sql = "select * from " + DBConstant::RELATIONAL_PREFIX + table_.GetTableName() + + "_log where hash_key = ?;"; sqlite3_stmt *queryStmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, queryStmt); if (errCode != E_OK) { @@ -630,17 +509,13 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen } // bind - SQLiteUtils::BindInt64ToStatement(statement, 1, dataKeyBind); - + SQLiteUtils::BindInt64ToStatement(statement, 1, dataKeyBind); // 1 means dataKey index std::vector originDev(logInfoBind.originDev.begin(), logInfoBind.originDev.end()); - SQLiteUtils::BindBlobToStatement(statement, 2, originDev); - - SQLiteUtils::BindInt64ToStatement(statement, 3, logInfoBind.timestamp); - SQLiteUtils::BindInt64ToStatement(statement, 4, logInfoBind.wTimeStamp); - SQLiteUtils::BindInt64ToStatement(statement, 5, logInfoBind.flag); - - SQLiteUtils::BindTextToStatement(statement, 6, logInfoBind.hashKey); - + SQLiteUtils::BindBlobToStatement(statement, 2, originDev); // 2 means ori_dev index + SQLiteUtils::BindInt64ToStatement(statement, 3, logInfoBind.timestamp); // 3 means timestamp index + SQLiteUtils::BindInt64ToStatement(statement, 4, logInfoBind.wTimeStamp); // 4 means w_timestamp index + SQLiteUtils::BindInt64ToStatement(statement, 5, logInfoBind.flag); // 5 means flag index + SQLiteUtils::BindTextToStatement(statement, 6, logInfoBind.hashKey); // 6 means hashKey index errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { return E_OK; @@ -651,7 +526,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem) { std::string devName = DBCommon::TransferHashString(dataItem.dev); - const std::string tableName = "distributeddatamgr_aux_" + table_.GetTableName() + "_" + + const std::string tableName = DBConstant::RELATIONAL_PREFIX + table_.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); std::string hashKey = std::string(dataItem.hashKey.begin(), dataItem.hashKey.end()); std::string sql = "DELETE FROM " + tableName + " WHERE calc_hash(" + table_.GetPrimaryKey() + ")=" + hashKey + ";"; @@ -694,12 +569,10 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncDataItem(sqlite3_stmt *sta for (size_t index = 0; index < data.optionalData.size(); index++) { const auto &filedData = data.optionalData[index]; - if (filedData.has_value()) { - (void)BindDataValueByType(statement, filedData, - filedData.value().GetType(), fieldInfos[index].GetColumnId() + 1); - } else { - (void)BindDataValueByType(statement, filedData, - StorageType::STORAGE_TYPE_NULL, fieldInfos[index].GetColumnId() + 1); + errCode = BindDataValueByType(statement, filedData, fieldInfos[index].GetColumnId() + 1); + if (errCode != E_OK) { + LOGE("Bind data failed, errCode:%d, cid:%d.", errCode, fieldInfos[index].GetColumnId() + 1); + return errCode; } } @@ -771,41 +644,88 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncItems(const QueryObject &o return errCode; } -static int GetLogInfoStatement(sqlite3 *dbHandle, const std::string &tableName, - uint64_t beginTime, uint64_t endTime, sqlite3_stmt *&statement) +int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *stmt, DataItem &dataItem, + bool isGettingDeletedData) const { - std::string sql = "select * from distributeddatamgr_aux_" + tableName + - "_log where flag=0x02 AND timestamp>=? AND timestamp fieldInfos; + if (!isGettingDeletedData) { + for (const auto &col: table_.GetFields()) { + auto colType = col.second.GetStorageType(); + auto colId = col.second.GetColumnId() + 7; // 7 means the count of log table's column. + LOGD("[GetDataItemForSync] field:%s type:%d cid:%d", col.second.GetFieldName().c_str(), colType, colId); + DataValue value; + errCode = GetDataValueByType(stmt, value, colType, colId); + if (errCode != E_OK) { + return errCode; + } + fieldInfos.push_back(col.second); + data.rowData.push_back(value); + } } - errCode = SQLiteUtils::BindInt64ToStatement(statement, 2, endTime); + errCode = DataTransformer::SerializeDataItem(data, fieldInfos, dataItem); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + LOGE("relational data value transfer to kv fail"); } return errCode; } int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, - QueryObject query, const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const + const DataSizeSpecInfo &dataSizeInfo, std::function getStmt) { - sqlite3_stmt *logStatement = nullptr; - int errCode = GetLogInfoStatement(dbHandle_, query.GetTableName(), timeRange.first, timeRange.second, logStatement); - if (errCode == E_OK) { - errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); + sqlite3_stmt *stmt = nullptr; + bool isGettingDeletedData = false; + int errCode = getStmt(dbHandle_, stmt, isGettingDeletedData); + if (errCode != E_OK) { + return errCode; } - SQLiteUtils::ResetStatement(logStatement, true, errCode); + + size_t dataTotalSize = 0; + do { + DataItem item; + errCode = SQLiteUtils::StepWithRetry(stmt, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + errCode = GetDataItemForSync(stmt, item, isGettingDeletedData); + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); + errCode = -E_FINISHED; + break; + } else { + LOGE("Get sync data error:%d", errCode); + break; + } + + // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. + dataTotalSize += GetDataItemSerialSize(item, appendLength); + if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || + dataItems.size() >= dataSizeInfo.packetSize) { + errCode = -E_UNFINISHED; + break; + } else { + dataItems.push_back(std::move(item)); + } + } while (true); + + SQLiteUtils::ResetStatement(stmt, true, errCode); return errCode; } + +int SQLiteSingleVerRelationalStorageExecutor::CheckDBModeForRelational() +{ + std::string journalMode; + int errCode = SQLiteUtils::GetJournalMode(dbHandle_, journalMode); + if (errCode != E_OK || journalMode != "wal") { + LOGE("Not support journal mode %s for relational db, expect wal mode, %d", journalMode.c_str(), errCode); + return -E_NOT_SUPPORT; + } + return E_OK; +} } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h index 301204f61a909cb08b0af0d94146015c7b0295df..ba0057ae17186556081e6f0c12097512a8692b0b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -16,8 +16,9 @@ #define SQLITE_SINGLE_VER_RELATIONAL_STORAGE_EXECUTOR_H #ifdef RELATIONAL_STORE -#include "macro_utils.h" +#include "data_transformer.h" #include "db_types.h" +#include "macro_utils.h" #include "sqlite_utils.h" #include "sqlite_storage_executor.h" #include "relational_store_delegate.h" @@ -32,19 +33,17 @@ public: // Delete the copy and assign constructors DISABLE_COPY_ASSIGN_MOVE(SQLiteSingleVerRelationalStorageExecutor); - int CreateDistributedTable(const std::string &tableName, const RelationalStoreDelegate::TableOption &option); + int CreateDistributedTable(const std::string &tableName, TableInfo &table); int StartTransaction(TransactType type); int Commit(); int Rollback(); - int SetTableInfo(QueryObject query); + int SetTableInfo(const QueryObject &query); // For Get sync data - int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, QueryObject query, - const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const; - int GetDeletedSyncDataByTimestamp(std::vector &dataItems, size_t appendLength, - TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const; + int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, + const DataSizeSpecInfo &dataSizeInfo, std::function getStmt); // operation of meta data int GetKvData(const Key &key, Value &value) const; @@ -57,14 +56,15 @@ public: int SaveSyncItems(const QueryObject &object, std::vector &dataItems, const std::string &deviceName, TimeStamp &timeStamp); + int AnalysisRelationalSchema(const std::string &tableName, TableInfo &tableInfo); + + int CheckDBModeForRelational(); + private: int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData) const; - int GetSyncDataItems(std::vector &dataItems, sqlite3_stmt *statement, - size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const; - - int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem) const; + int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem, bool isGettingDeletedData) const; int SaveSyncDataItems(const QueryObject &object, std::vector &dataItems, const std::string &deviceName, TimeStamp &timeStamp); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp index b99b34f90d97a77d2764ebe1e555ee647ce7993d..8ae53a1ac9181c2e3942306f00dea7ea7f541880 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp @@ -206,7 +206,7 @@ int SQLiteSingleVerStorageEngine::AddSubscribeToMainDBInMigrate() int errCode = E_OK; auto handle = static_cast(FindExecutor(true, OperatePerm::NORMAL_PERM, errCode)); if (errCode != E_OK || handle == nullptr) { - LOGE("Get available executor for add subscribe failed. %s", errCode); + LOGE("Get available executor for add subscribe failed. %d", errCode); return errCode; } errCode = handle->StartTransaction(TransactType::IMMEDIATE); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index 9d3e9a2ff5d71d252f84c0a571eee3595046924b..dd637aaed09630bfa908fedce1f312402a38c794 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -22,6 +22,7 @@ #include "db_common.h" #include "db_errno.h" #include "parcel.h" +#include "platform_specific.h" #include "runtime_context.h" #include "sqlite_single_ver_storage_executor_sql.h" @@ -1030,10 +1031,11 @@ int SQLiteSingleVerStorageExecutor::Commit() return -E_INVALID_DB; } int errCode = SQLiteUtils::CommitTransaction(dbHandle_); - if (errCode == E_OK) { - isTransactionOpen_ = false; + if (errCode != E_OK) { + return CheckCorruptedStatus(errCode); } - return CheckCorruptedStatus(errCode); + isTransactionOpen_ = false; + return E_OK; } int SQLiteSingleVerStorageExecutor::Rollback() @@ -1298,9 +1300,9 @@ int SQLiteSingleVerStorageExecutor::PrepareForNotifyConflictAndObserver(DataItem LOGD("[SingleVerExe] Ignore the sync data."); if (isSyncMigrating_) { ResetForMigrateCacheData(); - return -E_IGNOR_DATA; + return -E_IGNORE_DATA; } - return ResetSaveSyncStatements(-E_IGNOR_DATA); + return ResetSaveSyncStatements(-E_IGNORE_DATA); } notify.dataStatus = JudgeSyncSaveType(dataItem, notify.getData, deviceInfo.deviceName, isHashKeyExisted); @@ -1343,7 +1345,7 @@ int SQLiteSingleVerStorageExecutor::SaveSyncDataItem(DataItem &dataItem, const D int errCode = PrepareForNotifyConflictAndObserver(dataItem, deviceInfo, notify); if (errCode != E_OK) { - if (errCode == -E_IGNOR_DATA) { + if (errCode == -E_IGNORE_DATA) { errCode = E_OK; } return errCode; @@ -2132,4 +2134,33 @@ int SQLiteSingleVerStorageExecutor::CheckIntegrity() const return SQLiteUtils::CheckIntegrity(dbHandle_, CHECK_DB_INTEGRITY_SQL); } + +int SQLiteSingleVerStorageExecutor::ForceCheckPoint() const +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + SQLiteUtils::ExecuteCheckPoint(dbHandle_); + return E_OK; +} + +uint64_t SQLiteSingleVerStorageExecutor::GetLogFileSize() const +{ + if (isMemDb_) { + return 0; + } + + const char *fileName = sqlite3_db_filename(dbHandle_, "main"); + if (fileName == nullptr) { + return 0; + } + std::string walName = std::string(fileName) + "-wal"; + uint64_t fileSize = 0; + int result = OS::CalFileSize(std::string(walName), fileSize); + if (result != E_OK) { + return 0; + } + LOGI("The log file size is %llu", fileSize); + return fileSize; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h index 021199aeda551972aa919cd6c535ca4e126152fe..ee97d0b2839a0bfe3fde0c39d2152557b6a9edba 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h @@ -243,6 +243,10 @@ public: int GetSyncDataWithQuery(const QueryObject &query, size_t appendLength, const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange, std::vector &dataItems) const; + int ForceCheckPoint() const; + + uint64_t GetLogFileSize() const; + private: struct SaveRecordStatements { sqlite3_stmt *queryStatement = nullptr; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp index 872e873613f4d6e68993aaf08d497bee2732be84..9559b8fb4e28c65381fe07f35639af400639dc33 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp @@ -760,7 +760,7 @@ int SQLiteSingleVerStorageExecutor::PutIntoConflictAndCommitForMigrateCache(Data int errCode = PrepareForNotifyConflictAndObserver(dataItem, deviceInfo, notify); if (errCode != E_OK) { errCode = (errCode == -E_NOT_FOUND ? E_OK : errCode); - if (errCode == -E_IGNOR_DATA) { + if (errCode == -E_IGNORE_DATA) { notify.dataStatus.isDefeated = true; errCode = E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp index 009400728092ddbd0dadeeefd75b44166e90db0a..0d6084c9a014e5e85fe2dda7a6408838761a32a4 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp @@ -81,6 +81,7 @@ int SQLiteStorageEngine::CreateNewExecutor(bool isWrite, StorageExecutor *&handl dbHandle = nullptr; return errCode; } + SQLiteUtils::ExecuteCheckPoint(dbHandle); isUpdated_ = true; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 999d6f49fb57a40bd54ef70a8311f8c13cb75dbe..b4f6efaffd9a076811b80815f9715eb1cb885dba 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -102,7 +102,7 @@ std::string GetTriggerModeString(TriggerModeEnum mode) } } -int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp) +int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp, bool setWal) { uint64_t flag = SQLITE_OPEN_URI | SQLITE_OPEN_READWRITE; if (properties.createIfNecessary) { @@ -114,7 +114,10 @@ int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&db return -E_INVALID_ARGS; } std::string defaultAttachCipher = DEFAULT_ATTACH_CIPHER + cipherName + ";"; - std::vector sqls {WAL_MODE_SQL, defaultAttachCipher, DEFAULT_ATTACH_KDF_ITER}; + std::vector sqls {defaultAttachCipher, DEFAULT_ATTACH_KDF_ITER}; + if (setWal) { + sqls.push_back(WAL_MODE_SQL); + } std::string fileUrl = DBConstant::SQLITE_URL_PRE + properties.uri; int errCode = sqlite3_open_v2(fileUrl.c_str(), &dbTemp, flag, nullptr); @@ -139,7 +142,7 @@ END: return errCode; } -int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db) +int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db, bool setWal) { { // Only for register the sqlite3 log callback @@ -150,7 +153,7 @@ int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db) } } sqlite3 *dbTemp = nullptr; - int errCode = CreateDataBase(properties, dbTemp); + int errCode = CreateDataBase(properties, dbTemp, setWal); if (errCode != E_OK) { goto END; } @@ -602,11 +605,11 @@ int AnalysisSchemaSqlAndTrigger(sqlite3 *db, const std::string &tableName, Table (void) SQLiteUtils::GetColumnTextValue(statement, 0, type); if (type == "table") { std::string createTableSql; - (void) SQLiteUtils::GetColumnTextValue(statement, 4, createTableSql); + (void) SQLiteUtils::GetColumnTextValue(statement, 4, createTableSql); // 4 means create table sql table.SetCreateTableSql(createTableSql); } else if (type == "trigger") { std::string triggerName; - (void) SQLiteUtils::GetColumnTextValue(statement, 1, triggerName); + (void) SQLiteUtils::GetColumnTextValue(statement, 1, triggerName); // 1 means trigger name table.AddTrigger(triggerName); } } else { @@ -636,9 +639,9 @@ int GetSchemaIndexList(sqlite3 *db, const std::string &tableName, std::vector(sqlite3_column_int64(statement, 3))); + field.SetNotNull(static_cast(sqlite3_column_int64(statement, 3))); // 3 means whether null index - (void) SQLiteUtils::GetColumnTextValue(statement, 4, tmpString); + (void) SQLiteUtils::GetColumnTextValue(statement, 4, tmpString); // 4 means default value index if (!tmpString.empty()) { field.SetDefaultValue(tmpString); } - if (sqlite3_column_int64(statement, 5)) { + if (sqlite3_column_int64(statement, 5)) { // 5 means primary key index table.SetPrimaryKey(field.GetFieldName()); } table.AddField(field); @@ -902,6 +905,56 @@ int SQLiteUtils::GetVersion(sqlite3 *db, int &version) return errCode; } +int SQLiteUtils::GetJournalMode(sqlite3 *db, std::string &mode) +{ + if (db == nullptr) { + return -E_INVALID_DB; + } + + std::string sql = "PRAGMA journal_mode;"; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_prepare(db, sql.c_str(), -1, &statement, nullptr); + if (errCode != SQLITE_OK || statement == nullptr) { + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + return errCode; + } + + if (sqlite3_step(statement) == SQLITE_ROW) { + errCode = SQLiteUtils::GetColumnTextValue(statement, 0, mode); + } else { + LOGE("[SqlUtil][GetJournal] Get db journal_mode failed."); + errCode = SQLiteUtils::MapSQLiteErrno(SQLITE_ERROR); + } + + SQLiteUtils::ResetStatement(statement, true, errCode); + return E_OK; +} + +int SQLiteUtils::GetSynchronousMode(sqlite3 *db, int &mode) +{ + if (db == nullptr) { + return -E_INVALID_DB; + } + + std::string sql = "PRAGMA synchronous;"; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_prepare(db, sql.c_str(), -1, &statement, nullptr); + if (errCode != SQLITE_OK || statement == nullptr) { + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + return errCode; + } + + if (sqlite3_step(statement) == SQLITE_ROW) { + mode = sqlite3_column_int(statement, 0); + } else { + LOGE("[SqlUtil][GetSynchronous] Get db synchronous failed."); + errCode = SQLiteUtils::MapSQLiteErrno(SQLITE_ERROR); + } + + SQLiteUtils::ResetStatement(statement, true, errCode); + return E_OK; +} + int SQLiteUtils::SetUserVer(const OpenDbProperties &properties, int version) { if (properties.uri.empty()) { @@ -1252,7 +1305,7 @@ int SQLiteUtils::RegisterGetSysTime(sqlite3 *db) int SQLiteUtils::CreateRelationalMetaTable(sqlite3 *db) { std::string sql = - "CREATE TABLE IF NOT EXISTS distributeddatamgr_aux_metadata(" \ + "CREATE TABLE IF NOT EXISTS " + DBConstant::RELATIONAL_PREFIX + "metadata(" \ "key BLOB PRIMARY KEY NOT NULL," \ "value BLOB);"; @@ -1267,21 +1320,21 @@ int SQLiteUtils::CreateRelationalMetaTable(sqlite3 *db) int SQLiteUtils::CreateRelationalLogTable(sqlite3 *db, const std::string &oriTableName) { std::string sql = - "CREATE TABLE IF NOT EXISTS distributeddatamgr_aux_" + oriTableName + "_log(" \ + "CREATE TABLE IF NOT EXISTS " + DBConstant::RELATIONAL_PREFIX + oriTableName + "_log(" \ "data_key INT NOT NULL," \ "device BLOB," \ "ori_device BLOB," \ "timestamp INT NOT NULL," \ "wtimestamp INT NOT NULL," \ "flag INT NOT NULL," \ - "hash_key BLOB PRIMARY KEY NOT NULL);"; + "hash_key BLOB NOT NULL," + "PRIMARY KEY(device,hash_key));"; int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); if (errCode != E_OK) { LOGE("[SQLite] execute create table sql failed"); - return errCode; } - return E_OK; + return errCode; } template @@ -1296,10 +1349,10 @@ static std::string string_format(const std::string& format, Args ... args) int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &table) { std::string insertTrigger = - "CREATE TRIGGER IF NOT EXISTS distributeddatamgr_%s_ON_INSERT AFTER INSERT \n" \ + "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_INSERT AFTER INSERT \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t INSERT OR REPLACE INTO distributeddatamgr_aux_%s_log \ + "\t INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + "%s_log \ (data_key, device, ori_device, timestamp, wtimestamp, flag, hash_key)" \ "VALUES (new.rowid, '%s', '%s', get_sys_time(), get_sys_time(), 0x02, calc_hash(new.%s));\n" \ "END;"; @@ -1307,19 +1360,19 @@ int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &tabl table.GetTableName().c_str(), table.GetDevId().c_str(), table.GetDevId().c_str(), table.GetPrimaryKey().c_str()); std::string updateTrigger = - "CREATE TRIGGER IF NOT EXISTS distributeddatamgr_%s_ON_UPDATE AFTER UPDATE \n" \ + "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_UPDATE AFTER UPDATE \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t UPDATE distributeddatamgr_aux_%s_log SET timestamp=get_sys_time(), device='%s' \ + "\t UPDATE " + DBConstant::RELATIONAL_PREFIX + "%s_log SET timestamp=get_sys_time(), device='%s' \ where hash_key=calc_hash(old.%s) and flag&0x10=0;\n" \ "END;"; updateTrigger = string_format(updateTrigger, table.GetTableName().c_str(), table.GetTableName().c_str(), table.GetTableName().c_str(), table.GetDevId().c_str(), table.GetPrimaryKey().c_str()); std::string deleteTrigger = - "CREATE TRIGGER IF NOT EXISTS distributeddatamgr_%s_ON_DELETE BEFORE DELETE \n" \ + "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_DELETE BEFORE DELETE \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t UPDATE distributeddatamgr_aux_%s_log set flag=0x03,timestamp=get_sys_time() \ + "\t UPDATE " + DBConstant::RELATIONAL_PREFIX + "%s_log set flag=0x03,timestamp=get_sys_time() \ WHERE hash_key=calc_hash(old.%s);\n" \ "END;"; deleteTrigger = string_format(deleteTrigger, table.GetTableName().c_str(), @@ -1350,9 +1403,52 @@ int SQLiteUtils::CreateSameStuTable(sqlite3 *db, const std::string &oriTableName int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); if (errCode != E_OK) { LOGE("[SQLite] execute create table sql failed"); + } + return errCode; +} + +int SQLiteUtils::CloneIndexes(sqlite3 *db, const std::string &oriTableName, const std::string &newTableName) +{ + std::string sql = + "SELECT 'CREATE ' || CASE WHEN il.'unique' THEN 'UNIQUE ' ELSE '' END || 'INDEX ' || '" + + DBConstant::RELATIONAL_PREFIX + "' || il.name || ' ON ' || '" + newTableName + + "' || '(' || GROUP_CONCAT(ii.name) || ');' " + "FROM sqlite_master AS m," + "pragma_index_list(m.name) AS il," + "pragma_index_info(il.name) AS ii " + "WHERE m.type='table' AND m.name='" + oriTableName + "' AND il.origin='c' " + "GROUP BY il.name;"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Prepare the clone sql failed:%d", errCode); return errCode; } - return E_OK; + + sql.clear(); + while (true) { + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + const unsigned char *indexSql = sqlite3_column_text(stmt, 0); + sql += std::string(reinterpret_cast(indexSql)); + continue; + } + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + (void)ResetStatement(stmt, true, errCode); + break; + } + + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + if (errCode != E_OK) { + LOGE("[SQLite] execute create table sql failed"); + } + return errCode; } int SQLiteUtils::RegisterFunction(sqlite3 *db, const std::string &funcName, int nArg, void *uData, TransactFunc &func) @@ -1921,4 +2017,41 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) sqlite3_free(eSql); return E_OK; } + +void SQLiteUtils::ExecuteCheckPoint(sqlite3 *db) +{ + if (db == nullptr) { + return; + } + + int chkResult = sqlite3_wal_checkpoint_v2(db, nullptr, SQLITE_CHECKPOINT_TRUNCATE, nullptr, nullptr); + LOGI("SQLite checkpoint result:%d", chkResult); +} + +int SQLiteUtils::CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool &isEmpty) +{ + if (db == nullptr) { + return -E_INVALID_ARGS; + } + + std::string cntSql = "SELECT min(rowid) FROM " + tableName + ";"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, cntSql, stmt); + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + if (sqlite3_column_type(stmt, 0) == SQLITE_NULL) { + isEmpty = true; + } else { + isEmpty = false; + } + errCode = E_OK; + } + + SQLiteUtils::ResetStatement(stmt, true, errCode); + return SQLiteUtils::MapSQLiteErrno(errCode); +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 94d7140010cec92e15b9bc74a1a5319ee507b2c4..ea6478b62655c9191eb4c97e37627ffd49947363 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -21,7 +21,6 @@ #include "sqlite_import.h" #include "db_types.h" -#include "iprocess_system_api_adapter.h" #include "schema_object.h" #include "types.h" #ifdef RELATIONAL_STORE @@ -53,15 +52,15 @@ std::string GetTriggerModeString(TriggerModeEnum mode); } struct OpenDbProperties { - std::string uri{}; + std::string uri {}; bool createIfNecessary = true; bool isMemDb = false; - std::vector sqls{}; + std::vector sqls {}; CipherType cipherType = CipherType::AES_256_GCM; - CipherPassword passwd{}; - std::string schema{}; - std::string subdir{}; - SecurityOption securityOpt{}; + CipherPassword passwd {}; + std::string schema {}; + std::string subdir {}; + SecurityOption securityOpt {}; int conflictReslovePolicy = DEFAULT_LAST_WIN; bool createDirByStoreIdOnly = false; }; @@ -69,7 +68,7 @@ struct OpenDbProperties { class SQLiteUtils { public: // Initialize the SQLiteUtils with the given properties. - static int OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db); + static int OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db, bool setWal = true); // Check the statement and prepare the new if statement is null static int GetStatement(sqlite3 *db, const std::string &sql, sqlite3_stmt *&statement); @@ -117,6 +116,10 @@ public: static int GetVersion(sqlite3 *db, int &version); + static int GetJournalMode(sqlite3 *db, std::string &mode); + + static int GetSynchronousMode(sqlite3 *db, int &mode); + static int SetUserVer(const OpenDbProperties &properties, int version); static int SetUserVer(sqlite3 *db, int version); @@ -168,15 +171,20 @@ public: static int CreateSameStuTable(sqlite3 *db, const std::string &oriTableName, const std::string &newTableName, bool isCopyData); + static int CloneIndexes(sqlite3 *db, const std::string &oriTableName, const std::string &newTableName); #endif static int DropTriggerByName(sqlite3 *db, const std::string &name); static int ExpandedSql(sqlite3_stmt *stmt, std::string &basicString); + static void ExecuteCheckPoint(sqlite3 *db); + + static int CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool &isEmpty); + private: - static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp); + static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp, bool setWal); static int SetBusyTimeout(sqlite3 *db, int timeout); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp index 5ae456f8f4c701ca2f362a24ac4455d0e5c90a0e..66c4735c4d2734d99503bde8fe357494a7a19f49 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -21,13 +21,13 @@ namespace DistributedDB { SyncAbleEngine::SyncAbleEngine(ISyncInterface *store) - : started_(false), + : syncer_(), + started_(false), store_(store) {} SyncAbleEngine::~SyncAbleEngine() -{ -} +{} // Start a sync action. int SyncAbleEngine::Sync(const ISyncer::SyncParma &parm) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp index cf78835c376ad53eddef0eb358811f86ce99ecbe..dd6f1094c3bb30b4fd37ea3820cef637e44cb5df 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp @@ -206,7 +206,7 @@ int GenericSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) } if (!param.wait) { std::lock_guard lockGuard(syncIdLock_); - syncIdList_.push_back((int)syncId); + syncIdList_.push_back(static_cast(syncId)); } if (operation->CheckIsAllFinished()) { operation->Finished(); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b866028f0828437be053dd53622e64b8fbfb8349 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "single_ver_data_message_schedule.h" + +#include "db_common.h" +#include "log_print.h" +#include "version.h" +#include "single_ver_data_sync.h" + +namespace DistributedDB { +SingleVerDataMessageSchedule::~SingleVerDataMessageSchedule() +{ + LOGD("~SingleVerDataMessageSchedule"); + ClearMsg(); +} + +void SingleVerDataMessageSchedule::Initialize(const std::string &label, const std::string &deviceId) +{ + label_ = label; + deviceId_ = deviceId; +} + +void SingleVerDataMessageSchedule::PutMsg(Message *inMsg) +{ + if (inMsg == nullptr) { + return; + } + std::lock_guard lock(queueLock_); + msgQueue_.push(inMsg); + isNeedReload_ = true; +} + +bool SingleVerDataMessageSchedule::IsNeedReloadQueue() +{ + std::lock_guard lock(queueLock_); + return isNeedReload_; +} + +Message *SingleVerDataMessageSchedule::MoveNextMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, + bool &isNeedContinue) +{ + uint32_t remoteVersion = context->GetRemoteSoftwareVersion(); + if (remoteVersion < SOFTWARE_VERSION_RELEASE_3_0) { + // just get last msg when remote version is < 103 or >=103 but just open db now + return GetLastMsgFromQueue(); + } + { + std::lock_guard lock(workingLock_); + if (isWorking_) { + isNeedContinue = false; + return nullptr; + } + isWorking_ = true; + } + ResetTimer(context); + UpdateMsgMap(); + Message *msg = GetMsgFromMap(isNeedHandle); + isNeedContinue = true; + if (msg == nullptr) { + std::lock_guard lock(workingLock_); + isWorking_ = false; + return nullptr; + } + return msg; +} + +void SingleVerDataMessageSchedule::ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, + const Message *inMsg) +{ + if (isNeedHandleStatus) { + const DataRequestPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + LOGE("[DataMsgSchedule] packet is nullptr"); + return; + } + uint64_t curPacketId = packet->GetPacketId(); + { + std::lock_guard lock(lock_); + finishedPacketId_ = curPacketId; + if (isNeedClearMap) { + ClearMsgMapWithNoLock(); + expectedSequenceId_ = 1; + } else { + LOGI("[DataMsgSchedule] DealMsg seqId=%u finishedPacketId=%llu ok,label=%s,dev=%s", expectedSequenceId_, + finishedPacketId_, label_.c_str(), STR_MASK(deviceId_)); + expectedSequenceId_++; + } + } + } + std::lock_guard lock(workingLock_); + isWorking_ = false; +} + +void SingleVerDataMessageSchedule::ClearMsg() +{ + StopTimer(); + ClearMsgQueue(); + ClearMsgMap(); +} + +void SingleVerDataMessageSchedule::UpdateMsgMap() +{ + std::queue msgTmpQueue; + { + std::lock_guard lock(queueLock_); + while (!msgQueue_.empty()) { + msgTmpQueue.push(msgQueue_.front()); + msgQueue_.pop(); + } + isNeedReload_ = false; + } + UpdateMsgMapInner(msgTmpQueue); +} + +void SingleVerDataMessageSchedule::UpdateMsgMapInner(std::queue &msgTmpQueue) +{ + // update msg map + std::lock_guard lock(lock_); + while (!msgTmpQueue.empty()) { + Message *msg = msgTmpQueue.front(); + msgTmpQueue.pop(); + if (msg == nullptr) { + continue; + } + const DataRequestPacket *packet = msg->GetObject(); + if (packet == nullptr) { + delete msg; + continue; + } + uint32_t sessionId = msg->GetSessionId(); + uint32_t sequenceId = msg->GetSequenceId(); + uint64_t packetId = packet->GetPacketId(); + if (prevSessionId_ != 0 && sessionId == prevSessionId_) { + LOGD("[DataMsgSchedule] recv prev sessionId msg,drop msg,label=%s,dev=%s", label_.c_str(), + STR_MASK(deviceId_)); + delete msg; + continue; + } + if (sessionId != currentSessionId_) { + // make sure all msg sessionId is same in msgMap + ClearMsgMapWithNoLock(); + prevSessionId_ = currentSessionId_; + currentSessionId_ = sessionId; + finishedPacketId_ = 0; + expectedSequenceId_ = 1; + } + if (messageMap_.count(sequenceId) > 0) { + const auto *cachePacket = messageMap_[sequenceId]->GetObject(); + if (cachePacket != nullptr) { + LOGD("[DataMsgSchedule] msg packetId=%llu,cachePacketId=%llu,label=%s,dev=%s", packetId, + cachePacket->GetPacketId(), label_.c_str(), STR_MASK(deviceId_)); + if (packetId != 0 && packetId < cachePacket->GetPacketId()) { + delete msg; + continue; + } + } + delete messageMap_[sequenceId]; + messageMap_[sequenceId] = nullptr; + } + messageMap_[sequenceId] = msg; + LOGD("[DataMsgSchedule] put into msgMap seqId=%llu,packetId=%llu,label=%s,dev=%s", sequenceId, packetId, + label_.c_str(), STR_MASK(deviceId_)); + } +} + +Message *SingleVerDataMessageSchedule::GetMsgFromMap(bool &isNeedHandle) +{ + isNeedHandle = true; + std::lock_guard lock(lock_); + while (!messageMap_.empty()) { + auto iter = messageMap_.begin(); + Message *msg = iter->second; + messageMap_.erase(iter); + const DataRequestPacket *packet = msg->GetObject(); + if (packet == nullptr) { + LOGE("[DataMsgSchedule] expected error"); + delete msg; + continue; + } + uint32_t sequenceId = msg->GetSequenceId(); + uint64_t packetId = packet->GetPacketId(); + if (sequenceId < expectedSequenceId_) { + uint64_t revisePacketId = finishedPacketId_ - (expectedSequenceId_ - 1 - sequenceId); + LOGI("[DataMsgSchedule] msg seqId=%llu less than exSeqId=%llu,pacId=%llu,revisePacId=%llu,label=%s,dev=%s", + sequenceId, expectedSequenceId_, packetId, revisePacketId, label_.c_str(), STR_MASK(deviceId_)); + if (packetId < revisePacketId) { + delete msg; + continue; + } + // means already handle the msg, and just send E_OK ack in dataSync + isNeedHandle = false; + return msg; + } + if (sequenceId == expectedSequenceId_) { + if (packetId < finishedPacketId_) { + LOGI("[DataMsgSchedule] drop msg seqId=%llu,packetId=%llu,label=%s,dev=%s", sequenceId, packetId, + label_.c_str(), STR_MASK(deviceId_)); + delete msg; + continue; + } + // if packetId == finishedPacketId_ need handle + // it will happened while watermark/need_abilitySync when last ack is missing + return msg; + } + // sequenceId > expectedSequenceId_, not need handle, put into map again + messageMap_[sequenceId] = msg; + return nullptr; + } + return nullptr; +} + +Message *SingleVerDataMessageSchedule::GetLastMsgFromQueue() +{ + std::lock_guard lock(queueLock_); + isNeedReload_ = false; + while (!msgQueue_.empty()) { + Message *msg = msgQueue_.front(); + msgQueue_.pop(); + if (msgQueue_.empty()) { // means last msg + return msg; + } + delete msg; + } + return nullptr; +} + +void SingleVerDataMessageSchedule::ClearMsgMap() +{ + std::lock_guard lock(lock_); + ClearMsgMapWithNoLock(); +} + +void SingleVerDataMessageSchedule::ClearMsgMapWithNoLock() +{ + LOGD("[DataMsgSchedule] begin to ClearMsgMapWithNoLock"); + for (auto &iter : messageMap_) { + delete iter.second; + iter.second = nullptr; + } + messageMap_.clear(); +} + +void SingleVerDataMessageSchedule::ClearMsgQueue() +{ + std::lock_guard lock(queueLock_); + while (!msgQueue_.empty()) { + Message *msg = msgQueue_.front(); + msgQueue_.pop(); + delete msg; + } +} + +void SingleVerDataMessageSchedule::StartTimer(SingleVerSyncTaskContext *context) +{ + std::lock_guard lock(lock_); + TimerId timerId = 0; + RefObject::IncObjRef(context); + TimerAction timeOutCallback = std::bind(&SingleVerDataMessageSchedule::TimeOut, this, std::placeholders::_1); + int errCode = RuntimeContext::GetInstance()->SetTimer(IDLE_TIME_OUT, timeOutCallback, + [context]() { + int errCode = RuntimeContext::GetInstance()->ScheduleTask([context]() { + RefObject::DecObjRef(context); + }); + if (errCode != E_OK) { + LOGE("[DataMsgSchedule] timer finalizer ScheduleTask,errCode=%d", errCode); + } + }, timerId); + if (errCode != E_OK) { + RefObject::DecObjRef(context); + LOGE("[DataMsgSchedule] timer ScheduleTask, errCode=%d", errCode); + return; + } + timerId_ = timerId; + LOGD("[DataMsgSchedule] StartTimer,TimerId=%llu", timerId_); +} + +void SingleVerDataMessageSchedule::StopTimer() +{ + TimerId timerId; + { + std::lock_guard lock(lock_); + LOGD("[DataMsgSchedule] StopTimer,remove TimerId=%llu", timerId_); + if (timerId_ == 0) { + return; + } + timerId = timerId_; + timerId_ = 0; + } + RuntimeContext::GetInstance()->RemoveTimer(timerId); +} + +void SingleVerDataMessageSchedule::ResetTimer(SingleVerSyncTaskContext *context) +{ + StopTimer(); + StartTimer(context); +} + +int SingleVerDataMessageSchedule::TimeOut(TimerId timerId) +{ + if (IsNeedReloadQueue()) { + LOGI("[DataMsgSchedule] new msg exists, no need to timeout handle"); + return E_OK; + } + { + std::lock_guard lock(workingLock_); + if (isWorking_) { + LOGI("[DataMsgSchedule] other thread is handle msg, no need to timeout handle"); + return E_OK; + } + } + { + std::lock_guard lock(lock_); + LOGI("[DataMsgSchedule] timeout handling, stop timerId_[%llu]", timerId, timerId_); + if (timerId == timerId_) { + ClearMsgMapWithNoLock(); + timerId_ = 0; + } + } + RuntimeContext::GetInstance()->RemoveTimer(timerId); + return E_OK; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h old mode 100755 new mode 100644 similarity index 32% rename from services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h index 461ac6465ce5a26cb3e30a7b62cdb0b6c1c71a83..bbd274eaa07b8ec0d1f67390537914599af2a549 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2022 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,57 +13,62 @@ * limitations under the License. */ -#ifndef SLIDING_WINDOW_RECEIVER_H -#define SLIDING_WINDOW_RECEIVER_H - +#ifndef SINGLE_VER_DATA_MESSAGE_SCHEDULE_H +#define SINGLE_VER_DATA_MESSAGE_SCHEDULE_H #include +#include +#include +#include #include "message.h" -#include "single_ver_data_sync.h" +#include "runtime_context.h" #include "single_ver_sync_task_context.h" namespace DistributedDB { -class SlidingWindowReceiver { +class SingleVerDataMessageSchedule { public: - SlidingWindowReceiver() = default; - ~SlidingWindowReceiver(); - DISABLE_COPY_ASSIGN_MOVE(SlidingWindowReceiver); - - int Initialize(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int Receive(Message *inMsg); - void Clear(); + SingleVerDataMessageSchedule() = default; + ~SingleVerDataMessageSchedule(); + void Initialize(const std::string &label, const std::string &deviceId); + void PutMsg(Message *inMsg); + bool IsNeedReloadQueue(); + Message *MoveNextMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, bool &isNeedContinue); + void ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, const Message *inMsg); + void ClearMsg(); private: - int PutMsg(Message *inMsg); - void DealMsg(); - int TimeOut(TimerId timerId); - void StartTimer(); + void UpdateMsgMap(); + void UpdateMsgMapInner(std::queue &msgTmpQueue); + Message *GetMsgFromMap(bool &isNeedHandle); + Message *GetLastMsgFromQueue(); + void ClearMsgMap(); + void ClearMsgMapWithNoLock(); + void ClearMsgQueue(); + void StartTimer(SingleVerSyncTaskContext *context); void StopTimer(); - static void StopTimer(TimerId timerId); - void ClearMap(); - void ResetInfo(); - int ErrHandle(uint32_t sequenceId, uint64_t packetId); - void SetEndField(bool isLastSequence, uint32_t sequenceId); + void ResetTimer(SingleVerSyncTaskContext *context); + // when timeout queue size is 0 because thread can move queue msg to map if isNeedReload which is + // actived when queue has new msg is true + // so only need clear map msg + int TimeOut(TimerId timerId); + static constexpr int IDLE_TIME_OUT = 5 * 60 * 1000; // 5min + std::mutex queueLock_; + std::queue msgQueue_; + bool isNeedReload_ = false; + // only one thread is deal msg + std::mutex workingLock_; + bool isWorking_ = false; + // first:sequenceId second:Message*, deal msg from low sequenceId to high sequenceId std::mutex lock_; - std::condition_variable workingTaskcv_; - // 0 is default invalid. - uint32_t sessionId_ = 0; - // first:sequenceId second:Message*, message data not deal. std::map messageMap_; - // 0 is has finished nothing; e.g. 3 is sequenceId 1 2 3 has finished, sequenceId 4 has not finished. - uint32_t hasFinishedMaxId_ = 0; - uint64_t hasFinishedPacketId_ = 0; - // 0 is idle - uint32_t workingId_ = 0; - // 0 is has not received the end packet now. - uint32_t endId_ = 0; - bool isWaterMarkErrHappened_ = false; - // timeout for idle wait packet + uint32_t prevSessionId_ = 0; // drop the msg if msg sessionId is prev sessionId. + uint32_t currentSessionId_ = 0; + uint64_t finishedPacketId_ = 0; // next msg packetId should larger than it + uint32_t expectedSequenceId_ = 0; // handle next msg which sequenceId is equal to it TimerId timerId_ = 0; - static constexpr int IDLE_TIME_OUT = 5 * 60 * 1000; // 5min - SingleVerSyncTaskContext *context_ = nullptr; - std::shared_ptr dataSync_; -}; -} // namespace DistributedDB -#endif // SLIDING_WINDOW_RECEIVER_H + std::string label_; + std::string deviceId_; +}; +} +#endif // SINGLE_VER_DATA_MESSAGE_SCHEDULE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index 46cd9692e0d2a77f98dffb80477adae70e136b59..05117ece34a6b740799aa558fd34d72c725f2386 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -74,9 +74,152 @@ int SingleVerDataSync::Initialize(ISyncInterface *inStorage, ICommunicator *inCo label.resize(3); // only show 3 Bytes enough label_ = DBCommon::VectorToHexString(label); deviceId_ = deviceId; + msgSchedule_.Initialize(label_, deviceId_); return E_OK; } +int SingleVerDataSync::SyncStart(int mode, SingleVerSyncTaskContext *context) +{ + std::lock_guard lock(lock_); + if (sessionId_ != 0) { // auto sync timeout resend + return ReSendData(context); + } + ResetSyncStatus(mode, context); + LOGI("[DataSync] SendStart,mode=%d,label=%s,device=%s", mode_, label_.c_str(), STR_MASK(deviceId_)); + int tmpMode = SyncOperation::TransferSyncMode(mode); + int errCode = E_OK; + if (tmpMode == SyncModeType::PUSH) { + errCode = PushStart(context); + } else if (tmpMode == SyncModeType::PUSH_AND_PULL) { + errCode = PushPullStart(context); + } else if (tmpMode == SyncModeType::PULL) { + errCode = PullRequestStart(context); + } else { + errCode = PullResponseStart(context); + } + if (context->IsSkipTimeoutError(errCode)) { + // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. + // just return to avoid higher pressure for send. + return E_OK; + } + if (errCode != E_OK) { + LOGE("[DataSync] SendStart errCode=%d", errCode); + return errCode; + } + if (tmpMode == SyncModeType::PUSH_AND_PULL && context->GetTaskErrCode() == -E_EKEYREVOKED) { + LOGE("wait for recv finished for push and pull mode"); + return -E_EKEYREVOKED; + } + return InnerSyncStart(context); +} + +int SingleVerDataSync::InnerSyncStart(SingleVerSyncTaskContext *context) +{ + while (true) { + if (windowSize_ <= 0 || isAllDataHasSent_) { + LOGD("[DataSync] InnerDataSync winSize=%d,isAllSent=%d,label=%s,device=%s", windowSize_, isAllDataHasSent_, + label_.c_str(), STR_MASK(deviceId_)); + return E_OK; + } + int mode = SyncOperation::TransferSyncMode(mode_); + if (mode == SyncModeType::PULL) { + LOGE("[DataSync] unexpected error"); + return -E_INVALID_ARGS; + } + int errCode; + context->IncSequenceId(); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { + errCode = PushStart(context); + } else { + errCode = PullResponseStart(context); + } + if ((mode == SyncModeType::PUSH_AND_PULL) && errCode == -E_EKEYREVOKED) { + LOGE("[DataSync] wait for recv finished,label=%s,device=%s", label_.c_str(), STR_MASK(deviceId_)); + isAllDataHasSent_ = true; + return -E_EKEYREVOKED; + } + if (context->IsSkipTimeoutError(errCode)) { + // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. + // just return to avoid higher pressure for send. + return E_OK; + } + if (errCode != E_OK) { + LOGE("[DataSync] InnerSend errCode=%d", errCode); + return errCode; + } + } + return E_OK; +} + +void SingleVerDataSync::InnerClearSyncStatus() +{ + sessionId_ = 0; + reSendMap_.clear(); + windowSize_ = 0; + maxSequenceIdHasSent_ = 0; + isAllDataHasSent_ = false; +} + +int SingleVerDataSync::TryContinueSync(SingleVerSyncTaskContext *context, const Message *message) +{ + if (message == nullptr) { + LOGE("[DataSync] AckRecv message nullptr"); + return -E_INVALID_ARGS; + } + const DataAckPacket *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value + uint32_t sessionId = message->GetSessionId(); + uint32_t sequenceId = message->GetSequenceId(); + + std::lock_guard lock(lock_); + LOGI("[DataSync] recv ack seqId=%d,packetId=%llu,winSize=%d,label=%s,dev=%s", sequenceId, packetId, windowSize_, + label_.c_str(), STR_MASK(deviceId_)); + if (sessionId != sessionId_) { + LOGI("[DataSync] ignore ack,sessionId is different"); + return E_OK; + } + if (reSendMap_.count(sequenceId) != 0) { + reSendMap_.erase(sequenceId); + windowSize_++; + } else { + LOGI("[DataSync] ack seqId not in map"); + return E_OK; + } + if (!isAllDataHasSent_) { + return InnerSyncStart(context); + } else if (reSendMap_.size() == 0) { + context->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); + InnerClearSyncStatus(); + return -E_FINISHED; + } + return E_OK; +} + +void SingleVerDataSync::ClearSyncStatus() +{ + std::lock_guard lock(lock_); + InnerClearSyncStatus(); +} + +int SingleVerDataSync::ReSendData(SingleVerSyncTaskContext *context) +{ + if (reSendMap_.empty()) { + LOGI("[DataSync] ReSend map empty"); + return -E_INTERNAL_ERROR; + } + uint32_t sequenceId = reSendMap_.begin()->first; + ReSendInfo reSendInfo = reSendMap_.begin()->second; + LOGI("[DataSync] ReSend mode=%d,start=%llu,end=%llu,delStart=%llu,delEnd=%llu,seqId=%d,packetId=%llu,windowsize=%d," + "label=%s,deviceId=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, sequenceId, reSendInfo.packetId, windowSize_, label_.c_str(), STR_MASK(deviceId_)); + DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, + reSendInfo.deleteBeginTime, reSendInfo.deleteEndTime, reSendInfo.packetId}; + return ReSend(context, dataReSendInfo); +} + std::string SingleVerDataSync::GetLocalDeviceName() { std::string deviceInfo; @@ -252,6 +395,26 @@ int SingleVerDataSync::SaveData(const SingleVerSyncTaskContext *context, const s return errCode; } +void SingleVerDataSync::ResetSyncStatus(int inMode, SingleVerSyncTaskContext *context) +{ + mode_ = inMode; + maxSequenceIdHasSent_ = 0; + isAllDataHasSent_ = false; + context->ReSetSequenceId(); + reSendMap_.clear(); + if (context->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { + windowSize_ = LOW_VERSION_WINDOW_SIZE; + } else { + windowSize_ = HIGH_VERSION_WINDOW_SIZE; + } + int mode = SyncOperation::TransferSyncMode(inMode); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL) { + sessionId_ = context->GetRequestSessionId(); + } else { + sessionId_ = context->GetResponseSessionId(); + } +} + TimeStamp SingleVerDataSync::GetMaxSendDataTime(const std::vector &inData) { TimeStamp stamp = 0; @@ -574,6 +737,38 @@ void SingleVerDataSync::TransSendDataItemToLocal(const SingleVerSyncTaskContext } } +void SingleVerDataSync::SetSessionEndTimeStamp(TimeStamp end) +{ + sessionEndTimeStamp_ = end; +} + +TimeStamp SingleVerDataSync::GetSessionEndTimeStamp() const +{ + return sessionEndTimeStamp_; +} + +void SingleVerDataSync::UpdateSendInfo(SyncTimeRange dataTimeRange, SingleVerSyncTaskContext *context) +{ + ReSendInfo reSendInfo; + reSendInfo.start = dataTimeRange.beginTime; + reSendInfo.end = dataTimeRange.endTime; + reSendInfo.deleteBeginTime = dataTimeRange.deleteBeginTime; + reSendInfo.deleteEndTime = dataTimeRange.deleteEndTime; + reSendInfo.packetId = context->GetPacketId(); + maxSequenceIdHasSent_++; + reSendMap_[maxSequenceIdHasSent_] = reSendInfo; + windowSize_--; + ContinueToken token; + context->GetContinueToken(token); + if (token == nullptr) { + isAllDataHasSent_ = true; + } + LOGI("[DataSync] mode=%d,start=%llu,end=%llu,deleteStart=%llu,deleteEnd=%llu,seqId=%d,packetId=%llu,window_size=%d," + "isAllSend=%d,label=%s,device=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, maxSequenceIdHasSent_, reSendInfo.packetId, windowSize_, isAllDataHasSent_, + label_.c_str(), STR_MASK(deviceId_)); +} + void SingleVerDataSync::FillDataRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, SyncEntry &syncData, int sendCode, int mode) { @@ -635,11 +830,7 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) SyncEntry syncData; // get data errCode = GetDataWithPerformanceRecord(context, syncData); - // once get data occur E_EKEYREVOKED error, should also send request to remote dev to pull data. - if (SyncOperation::TransferSyncMode(mode) == SyncModeType::PUSH_AND_PULL && - context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && errCode == -E_EKEYREVOKED) { - errCode = E_OK; - } + TranslateErrCodeIfNeed(mode, context->GetRemoteSoftwareVersion(), errCode); if (!IsGetDataSuccessfully(errCode)) { LOGE("[DataSync][PushStart] get data failed, errCode=%d", errCode); @@ -654,9 +845,8 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; UpdateWaterMark isUpdateWaterMark; SyncTimeRange dataTime = GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); - context->SetSequenceStartAndEndTimeStamp(dataTime); if (errCode == E_OK) { - context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); + SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } FillDataRequestPacket(packet, context, syncData, errCode, mode); errCode = SendDataPacket(curType, packet, context); @@ -664,6 +854,9 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) if (performance != nullptr) { performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_MACHINE_START_TO_PUSH_SEND); } + if (errCode == E_OK || errCode == -E_TIMEOUT) { + UpdateSendInfo(dataTime, context); + } if (errCode == E_OK) { if (curType == SyncType::QUERY_SYNC_TYPE && (context->GetQuery().HasLimit() || context->GetQuery().HasOrderBy())) { @@ -676,6 +869,15 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) return errCode; } +void SingleVerDataSync::TranslateErrCodeIfNeed(int mode, uint32_t version, int &errCode) +{ + // once get data occur E_EKEYREVOKED error, should also send request to remote dev to pull data. + if (SyncOperation::TransferSyncMode(mode) == SyncModeType::PUSH_AND_PULL && version > SOFTWARE_VERSION_RELEASE_2_0 + && errCode == -E_EKEYREVOKED) { + errCode = E_OK; + } +} + int SingleVerDataSync::PushStart(SingleVerSyncTaskContext *context) { if (context == nullptr) { @@ -723,7 +925,6 @@ int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); WaterMark endMark = context->GetEndMark(); SyncTimeRange dataTime = {localMark, deleteMark, localMark, deleteMark}; - context->SetSequenceStartAndEndTimeStamp(dataTime); packet->SetBasicInfo(E_OK, version, context->GetMode()); packet->SetWaterMark(localMark, peerMark, deleteMark); @@ -736,6 +937,7 @@ int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) LOGD("[DataSync][Pull] curType=%d,local=%llu,del=%llu,end=%llu,peer=%llu,label=%s,dev=%s", syncType, localMark, deleteMark, peerMark, endMark, label_.c_str(), STR_MASK(GetDeviceId())); + UpdateSendInfo(dataTime, context); return SendDataPacket(syncType, packet, context); } @@ -763,13 +965,14 @@ int SingleVerDataSync::PullResponseStart(SingleVerSyncTaskContext *context) } SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; UpdateWaterMark isUpdateWaterMark; - SyncTimeRange dataTime = - GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); - context->SetSequenceStartAndEndTimeStamp(dataTime); + SyncTimeRange dataTime = GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); if (errCode == E_OK) { - context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); + SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } errCode = SendPullResponseDataPkt(ackCode, syncData, context); + if (errCode == E_OK || errCode == -E_TIMEOUT) { + UpdateSendInfo(dataTime, context); + } if (errCode == E_OK) { if (curType == SyncType::QUERY_SYNC_TYPE && (context->GetQuery().HasLimit() || context->GetQuery().HasOrderBy())) { @@ -1016,6 +1219,11 @@ int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, SyncEntry &syncOutDa return errCode; } +void SingleVerDataSync::SendFinishedDataAck(SingleVerSyncTaskContext *context, const Message *message) +{ + SendDataAck(context, message, E_OK, 0); +} + int SingleVerDataSync::SendDataAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, WaterMark maxSendDataTime) { @@ -1048,6 +1256,32 @@ int SingleVerDataSync::SendDataAck(SingleVerSyncTaskContext *context, const Mess return errCode; } +bool SingleVerDataSync::AckPacketIdCheck(const Message *message) +{ + if (message == nullptr) { + LOGE("[DataSync] AckRecv message nullptr"); + return false; + } + if (message->GetMessageType() == TYPE_NOTIFY || message->IsFeedbackError()) { + return false; + } + const DataAckPacket *packet = message->GetObject(); + if (packet == nullptr) { + return false; + } + uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value + std::lock_guard lock(lock_); + uint32_t sequenceId = message->GetSequenceId(); + if (reSendMap_.count(sequenceId) != 0) { + uint64_t originalPacketId = reSendMap_[sequenceId].packetId; + if (DataAckPacket::IsPacketIdValid(packetId) && packetId != originalPacketId) { + LOGE("[DataSync] packetId[%llu] is not match with original[%llu]", packetId, originalPacketId); + return false; + } + } + return true; +} + int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message *message) { int errCode = AckMsgErrnoCheck(context, message); @@ -1067,8 +1301,7 @@ int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message } if (recvCode == -E_NEED_ABILITY_SYNC || recvCode == -E_NOT_PERMIT) { - // after set sliding window sender err, we can ReleaseContinueToken, avoid crash - context->SetSlidingWindowSenderErr(true); + // we should ReleaseContinueToken, avoid crash LOGI("[DataSync][AckRecv] Data sync abort,recvCode =%d,label =%s,dev=%s", recvCode, label_.c_str(), STR_MASK(GetDeviceId())); context->ReleaseContinueToken(); @@ -1158,9 +1391,6 @@ void SingleVerDataSync::GetPullEndWatermark(const SingleVerSyncTaskContext *cont int SingleVerDataSync::DealWaterMarkException(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, const std::vector &reserved) { - // after set sliding window sender err, we can SaveLocalWaterMark, avoid sliding window sender re save a wrong - // waterMark again. - context->SetSlidingWindowSenderErr(true); WaterMark deletedWaterMark = 0; SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; if (curType == SyncType::QUERY_SYNC_TYPE) { @@ -1660,12 +1890,12 @@ void SingleVerDataSync::FillRequestReSendPacket(const SingleVerSyncTaskContext * // transer reSend mode, RESPONSE_PULL transfer to push or query push // PUSH_AND_PULL mode which sequenceId lager than first transfer to push or query push int reSendMode = GetReSendMode(context->GetMode(), reSendInfo.sequenceId, curType); - if (context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) || + if (GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) || SyncOperation::TransferSyncMode(context->GetMode()) == SyncModeType::PULL) { LOGI("[DataSync][ReSend] set lastid,label=%s,dev=%s", label_.c_str(), STR_MASK(GetDeviceId())); packet->SetLastSequence(); } - if (sendCode == E_OK && context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) && + if (sendCode == E_OK && GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) && context->GetMode() == SyncModeType::RESPONSE_PULL) { sendCode = SEND_FINISHED; } @@ -2148,4 +2378,30 @@ std::string SingleVerDataSync::GetDeleteSyncId(const SingleVerSyncTaskContext *c #endif return id; } + +void SingleVerDataSync::PutDataMsg(Message *message) +{ + return msgSchedule_.PutMsg(message); +} + +Message *SingleVerDataSync::MoveNextDataMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, + bool &isNeedContinue) +{ + return msgSchedule_.MoveNextMsg(context, isNeedHandle, isNeedContinue); +} + +bool SingleVerDataSync::IsNeedReloadQueue() +{ + return msgSchedule_.IsNeedReloadQueue(); +} + +void SingleVerDataSync::ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, const Message *message) +{ + msgSchedule_.ScheduleInfoHandle(isNeedHandleStatus, isNeedClearMap, message); +} + +void SingleVerDataSync::ClearDataMsg() +{ + msgSchedule_.ClearMsg(); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h index bcb2b0b973954280de5e408e9bd3c02c4ad3acac..8788197e7fa431f387f09ecc5463e116b80a260f 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h @@ -20,6 +20,7 @@ #include "isync_interface.h" #include "meta_data.h" #include "parcel.h" +#include "single_ver_data_message_schedule.h" #include "single_ver_data_packet.h" #include "single_ver_kvdb_sync_interface.h" #include "single_ver_sync_task_context.h" @@ -29,6 +30,15 @@ namespace DistributedDB { using SendDataItem = SingleVerKvEntry *; +struct ReSendInfo { + TimeStamp start = 0; + TimeStamp end = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp deleteEndTime = 0; + // packetId is used for matched ackpacket packetId which saved in ackPacket.reserve + // if equaled, means need to handle the ack, or drop. it is always increased + uint64_t packetId = 0; +}; struct DataSyncReSendInfo { uint32_t sessionId = 0; @@ -55,6 +65,12 @@ public: int Initialize(ISyncInterface *inStorage, ICommunicator *inCommunicateHandle, std::shared_ptr &inMetadata, const std::string &deviceId); + int SyncStart(int mode, SingleVerSyncTaskContext *context); + + int TryContinueSync(SingleVerSyncTaskContext *context, const Message *message); + + void ClearSyncStatus(); + int PushStart(SingleVerSyncTaskContext *context); int PushPullStart(SingleVerSyncTaskContext *context); @@ -65,6 +81,8 @@ public: int DataRequestRecv(SingleVerSyncTaskContext *context, const Message *message, WaterMark &pullEndWatermark); + bool AckPacketIdCheck(const Message *message); + int AckRecv(SingleVerSyncTaskContext *context, const Message *message); void SendSaveDataNotifyPacket(SingleVerSyncTaskContext *context, uint32_t pktVersion, uint32_t sessionId, @@ -73,8 +91,6 @@ public: virtual int SendDataAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, WaterMark maxSendDataTime); - int32_t ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo); - int CheckPermitSendData(int inMode, SingleVerSyncTaskContext *context); std::string GetLabel() const; @@ -94,6 +110,18 @@ public: void ControlAckErrorHandle(const SingleVerSyncTaskContext *context, const std::shared_ptr &subManager) const; + void PutDataMsg(Message *message); + + Message *MoveNextDataMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, bool &isNeedContinue); + + bool IsNeedReloadQueue(); + + void SendFinishedDataAck(SingleVerSyncTaskContext *context, const Message *message); + + void ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, const Message *message); + + void ClearDataMsg(); + protected: static const int SEND_FINISHED = 0xff; static const int LOCAL_WATER_MARK_NOT_INIT = 0xaa; @@ -101,15 +129,31 @@ protected: static const int WATER_MARK_INVALID = 0xbb; static const int MTU_SIZE = 28311552; // 27MB + void ResetSyncStatus(int inMode, SingleVerSyncTaskContext *context); + + int InnerSyncStart(SingleVerSyncTaskContext *context); + + void InnerClearSyncStatus(); + + int ReSendData(SingleVerSyncTaskContext *context); + + int32_t ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo); + TimeStamp GetMaxSendDataTime(const std::vector &inData); TimeStamp GetMinSendDataTime(const std::vector &inData, WaterMark localMark); + void SetSessionEndTimeStamp(TimeStamp end); + + TimeStamp GetSessionEndTimeStamp() const; + void FillDataRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, SyncEntry &syncData, int sendCode, int mode); int RequestStart(SingleVerSyncTaskContext *context, int mode); + void TranslateErrCodeIfNeed(int mode, uint32_t version, int &errCode); + SyncTimeRange GetSyncDataTimeRange(SyncType syncType, SingleVerSyncTaskContext *context, const std::vector &inData, UpdateWaterMark &isUpdate); @@ -220,6 +264,8 @@ protected: int GetReSendMode(int mode, uint32_t sequenceId, SyncType syncType); + void UpdateSendInfo(SyncTimeRange dataTimeRange, SingleVerSyncTaskContext *context); + void FillRequestReSendPacket(const SingleVerSyncTaskContext *context, DataRequestPacket *packet, DataSyncReSendInfo reSendInfo, SyncEntry &syncData, int sendCode); @@ -260,8 +306,25 @@ protected: std::shared_ptr metadata_; std::string label_; std::string deviceId_; + + SingleVerDataMessageSchedule msgSchedule_; + + static const int HIGH_VERSION_WINDOW_SIZE = 3; + static const int LOW_VERSION_WINDOW_SIZE = 1; + // below param is about sliding sync info, is different from every sync task + std::mutex lock_; + int mode_ = 0; // sync mode, may diff from context mode if trigger pull_response while push finish + uint32_t sessionId_ = 0; + // sequenceId as key + std::map reSendMap_; + // remaining sending window + int32_t windowSize_ = 0; + // max sequenceId has been sent + uint32_t maxSequenceIdHasSent_ = 0; + bool isAllDataHasSent_ = false; + // in a sync session, the last data timeStamp + TimeStamp sessionEndTimeStamp_ = 0; }; } // namespace DistributedDB #endif // SINGLE_VER_DATA_SYNC_NEW_H - diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp deleted file mode 100755 index 4b9926ac647d16271b4a86a463484871d96eef60..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "single_ver_data_sync_with_sliding_window.h" - -namespace DistributedDB { -int SingleVerDataSyncWithSlidingWindow::SenderStart(int32_t mode, SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - return sender_.SendStart(mode, context, dataSync); -} - -int SingleVerDataSyncWithSlidingWindow::SenderAckRecv(const Message *message) -{ - return sender_.AckRecv(message); -} - -int SingleVerDataSyncWithSlidingWindow::PreHandleSenderAckRecv(const Message *message) -{ - return sender_.PreHandleAckRecv(message); -} - -void SingleVerDataSyncWithSlidingWindow::SenderClear() -{ - sender_.Clear(); -} - -void SingleVerDataSyncWithSlidingWindow::SetSenderErr(bool isErr) -{ - sender_.SetErr(isErr); -} - -int SingleVerDataSyncWithSlidingWindow::ReceiverInit(SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - return receiver_.Initialize(context, dataSync); -} - -int SingleVerDataSyncWithSlidingWindow::SenderInit(SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - return sender_.Initialize(context, dataSync); -} - -void SingleVerDataSyncWithSlidingWindow::ReceiverClear() -{ - receiver_.Clear(); -} - -int SingleVerDataSyncWithSlidingWindow::Receive(Message *inMsg) -{ - return receiver_.Receive(inMsg); -} -} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp index 95396a1160d8e616258185b0cd0cc803b611e87a..d1c8732ac81e6bf41c1cb62c68bebf1f78ad76c2 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp @@ -160,7 +160,7 @@ void SingleVerKVSyncer::RemoteDataChanged(const std::string &device) void SingleVerKVSyncer::QueryAutoSync(const InternalSyncParma ¶m) { - LOGI("[SingleVerKVSyncer] trigger query sync_type=%u,dev=%s", param.mode, GetSyncDevicesStr(param.devices).c_str()); + LOGI("[SingleVerKVSyncer] trigger query syncmode=%u,dev=%s", param.mode, GetSyncDevicesStr(param.devices).c_str()); RefObject::IncObjRef(syncEngine_); int retCode = RuntimeContext::GetInstance()->ScheduleTask([this, param] { int errCode = Sync(param); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp index 31fc0d11482587a21a8e033ea65ef83c591bf528..7cb09883fc8e5fa0191d9ca3e153df48956edf20 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp @@ -1,154 +1,153 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "single_ver_relational_syncer.h" -#ifdef RELATIONAL_STORE -#include "db_common.h" -#include "relational_db_sync_interface.h" -#include "single_ver_sync_engine.h" - -namespace DistributedDB { -int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) -{ - const auto &syncInterface = static_cast(syncInterface_); - std::vector tablesQuery; - if (param.isQuerySync) { - tablesQuery.push_back(param.syncQuery); - } else { - tablesQuery = syncInterface->GetTablesQuery(); - } - std::set subSyncIdSet; - int errCode = GenerateEachSyncTask(param, syncId, tablesQuery, subSyncIdSet); - if (errCode != E_OK) { - DoRollBack(subSyncIdSet); - return errCode; - } - return E_OK; -} - -int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, - const std::vector &tablesQuery, std::set &subSyncIdSet) -{ - SyncParma subParam = param; - subParam.isQuerySync = true; - int errCode = E_OK; - for (const QuerySyncObject &table : tablesQuery) { - uint32_t subSyncId = GenerateSyncId(); - LOGI("[SingleVerRelationalSyncer] SubSyncId %d create by SyncId %d, tableName = %s", - subSyncId, syncId, table.GetRelationTableName().c_str()); - subParam.syncQuery = table; - subParam.onComplete = std::bind(&SingleVerRelationalSyncer::DoOnSubSyncComplete, this, subSyncId, - syncId, param, std::placeholders::_1); - { - std::lock_guard lockGuard(syncMapLock_); - fullSyncIdMap_[syncId].insert(subSyncId); - } - errCode = GenericSyncer::PrepareSync(subParam, subSyncId); - if (errCode != E_OK) { - LOGW("[SingleVerRelationalSyncer] PrepareSync failed errCode:%d", errCode); - std::lock_guard lockGuard(syncMapLock_); - fullSyncIdMap_[syncId].erase(subSyncId); - break; - } - subSyncIdSet.insert(subSyncId); - } - return errCode; -} - -void SingleVerRelationalSyncer::DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, - const SyncParma ¶m, const std::map &devicesMap) -{ - static std::map statusMap = { - { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, - { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, - { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, - { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, - { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, - { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, - { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, - { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, - { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, - { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, - { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, - }; - bool allFinish = true; - { - std::lock_guard lockGuard(syncMapLock_); - fullSyncIdMap_[syncId].erase(subSyncId); - allFinish = fullSyncIdMap_[syncId].empty(); - for (const auto &item : devicesMap) { - DBStatus status = DB_ERROR; - if (statusMap.find(item.second) != statusMap.end()) { - status = statusMap[item.second]; - } - resMap_[syncId][item.first][param.syncQuery.GetRelationTableName()] = status; - } - } - if (allFinish) { - DoOnComplete(param, syncId); - } -} - -void SingleVerRelationalSyncer::DoRollBack(std::set &subSyncIdSet) -{ - for (const auto &removeId : subSyncIdSet) { - int retCode = RemoveSyncOperation((int)removeId); - if (retCode != E_OK) { - LOGW("[SingleVerRelationalSyncer] RemoveSyncOperation failed errCode:%d, syncId:%d", retCode, removeId); - } - } -} - -void SingleVerRelationalSyncer::DoOnComplete(const SyncParma ¶m, uint32_t syncId) -{ - if (!param.relationOnComplete) { - return; - } - std::map> syncRes; - std::map> tmpMap; - { - std::lock_guard lockGuard(syncMapLock_); - tmpMap = resMap_[syncId]; - } - for (const auto &devicesRes : tmpMap) { - for (const auto &tableRes : devicesRes.second) { - syncRes[devicesRes.first].push_back( - {tableRes.first, static_cast(tableRes.second)}); - } - } - param.relationOnComplete(syncRes); - { - std::lock_guard lockGuard(syncMapLock_); - resMap_.erase(syncId); - } -} - -void SingleVerRelationalSyncer::EnableAutoSync(bool enable) -{ -} - -int SingleVerRelationalSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) -{ - return E_OK; -} - -void SingleVerRelationalSyncer::LocalDataChanged(int notifyEvent) -{ -} - -void SingleVerRelationalSyncer::RemoteDataChanged(const std::string &device) -{ -} -} +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "single_ver_relational_syncer.h" +#ifdef RELATIONAL_STORE +#include "db_common.h" +#include "relational_db_sync_interface.h" +#include "single_ver_sync_engine.h" + +namespace DistributedDB { +int SingleVerRelationalSyncer::Sync(const SyncParma ¶m) +{ + if (param.mode == SYNC_MODE_PUSH_PULL) { + return -E_NOT_SUPPORT; + } + if (param.syncQuery.GetRelationTableName().empty()) { + return -E_NOT_SUPPORT; + } + return GenericSyncer::Sync(param); +} + +int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) +{ + const auto &syncInterface = static_cast(syncInterface_); + std::vector tablesQuery; + if (param.isQuerySync) { + tablesQuery.push_back(param.syncQuery); + } else { + tablesQuery = syncInterface->GetTablesQuery(); + } + std::set subSyncIdSet; + int errCode = GenerateEachSyncTask(param, syncId, tablesQuery, subSyncIdSet); + if (errCode != E_OK) { + DoRollBack(subSyncIdSet); + return errCode; + } + if (param.wait) { + DoOnComplete(param, syncId); + } + return E_OK; +} + +int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, + const std::vector &tablesQuery, std::set &subSyncIdSet) +{ + SyncParma subParam = param; + subParam.isQuerySync = true; + int errCode = E_OK; + for (const QuerySyncObject &table : tablesQuery) { + uint32_t subSyncId = GenerateSyncId(); + LOGI("[SingleVerRelationalSyncer] SubSyncId %d create by SyncId %d, tableName = %s", + subSyncId, syncId, table.GetRelationTableName().c_str()); + subParam.syncQuery = table; + subParam.onComplete = std::bind(&SingleVerRelationalSyncer::DoOnSubSyncComplete, this, subSyncId, + syncId, param, std::placeholders::_1); + { + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].insert(subSyncId); + } + errCode = GenericSyncer::PrepareSync(subParam, subSyncId); + if (errCode != E_OK) { + LOGW("[SingleVerRelationalSyncer] PrepareSync failed errCode:%d", errCode); + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].erase(subSyncId); + break; + } + subSyncIdSet.insert(subSyncId); + } + return errCode; +} + +void SingleVerRelationalSyncer::DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, + const SyncParma ¶m, const std::map &devicesMap) +{ + bool allFinish = true; + { + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].erase(subSyncId); + allFinish = fullSyncIdMap_[syncId].empty(); + for (const auto &item : devicesMap) { + resMap_[syncId][item.first][param.syncQuery.GetRelationTableName()] = static_cast(item.second); + } + } + // block sync do callback in sync function + if (allFinish && !param.wait) { + DoOnComplete(param, syncId); + } +} + +void SingleVerRelationalSyncer::DoRollBack(std::set &subSyncIdSet) +{ + for (const auto &removeId : subSyncIdSet) { + int retCode = RemoveSyncOperation(static_cast(removeId)); + if (retCode != E_OK) { + LOGW("[SingleVerRelationalSyncer] RemoveSyncOperation failed errCode:%d, syncId:%d", retCode, removeId); + } + } +} + +void SingleVerRelationalSyncer::DoOnComplete(const SyncParma ¶m, uint32_t syncId) +{ + if (!param.relationOnComplete) { + return; + } + std::map> syncRes; + std::map> tmpMap; + { + std::lock_guard lockGuard(syncMapLock_); + tmpMap = resMap_[syncId]; + } + for (const auto &devicesRes : tmpMap) { + for (const auto &tableRes : devicesRes.second) { + syncRes[devicesRes.first].push_back( + {tableRes.first, static_cast(tableRes.second)}); + } + } + param.relationOnComplete(syncRes); + { + std::lock_guard lockGuard(syncMapLock_); + resMap_.erase(syncId); + fullSyncIdMap_.erase(syncId); + } +} + +void SingleVerRelationalSyncer::EnableAutoSync(bool enable) +{ +} + +int SingleVerRelationalSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +{ + return E_OK; +} + +void SingleVerRelationalSyncer::LocalDataChanged(int notifyEvent) +{ +} + +void SingleVerRelationalSyncer::RemoteDataChanged(const std::string &device) +{ +} +} #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h index 159a011eaadaae1938b108c1be16add91083fc17..15e4ef814348398e2d277300316edc6abf7bdea9 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h @@ -22,6 +22,9 @@ public: SingleVerRelationalSyncer() = default; ~SingleVerRelationalSyncer() override = default; + // Sync function. use SyncParma to reduce paramter. + int Sync(const SyncParma ¶m) override; + void EnableAutoSync(bool enable) override; int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; @@ -50,6 +53,5 @@ private: std::map>> resMap_; }; } - #endif -#endif // RELATIONAL_SYNCER_H +#endif // RELATIONAL_SYNCER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp deleted file mode 100644 index ea78cbe1b0f044cb39ed574c1388b11c07a7f03c..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "single_ver_subscribe_manager.h" -#include "db_common.h" -#include "sync_types.h" - -namespace DistributedDB { -void SingleVerSubscribeManager::ClearRemoteSubscribeQuery(const std::string &device) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - ClearSubscribeQuery(device, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::ClearAllRemoteQuery() -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - remoteSubscribedMap_.clear(); - remoteSubscribedTotalMap_.clear(); -} - -void SingleVerSubscribeManager::ClearLocalSubscribeQuery(const std::string &device) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - unFinishedLocalAutoSubMap_.erase(device); - ClearSubscribeQuery(device, localSubscribeMap_, localSubscribeTotalMap_); -} - -int SingleVerSubscribeManager::ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - int errCode = ReserveSubscribeQuery(device, query, remoteSubscribedMap_, remoteSubscribedTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote reserve err=%d", - STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - int errCode = ActiveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote active err=%d", - STR_MASK(device), STR_MASK(queryId), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - int errCode = ReserveSubscribeQuery(device, query, localSubscribeMap_, localSubscribeTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local reserve err=%d", - STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - int errCode = ActiveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local active err=%d", - STR_MASK(device), STR_MASK(queryId), errCode); - if (errCode != E_OK) { - return errCode; - } - if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && - unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { - unFinishedLocalAutoSubMap_[device].erase(queryId); - } - return errCode; -} - -void SingleVerSubscribeManager::DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - DeleteSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); -} - -void SingleVerSubscribeManager::DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - DeleteSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::PutLocalUnFiniedSubQueries(const std::string &device, - std::vector &subscribeQueries) -{ - LOGI("[SingleVerSubscribeManager] put local unfinished subscribe queries, nums=%d", subscribeQueries.size()); - std::unique_lock lockGuard(localSubscribeMapLock_); - if (subscribeQueries.size() == 0) { - unFinishedLocalAutoSubMap_.erase(device); - return; - } - unFinishedLocalAutoSubMap_[device].clear(); - auto iter = unFinishedLocalAutoSubMap_.find(device); - for (const auto &query : subscribeQueries) { - iter->second.insert(query.GetIdentify()); - } -} - -void SingleVerSubscribeManager::GetAllUnFinishSubQueries( - std::map> &allSyncQueries) const -{ - std::shared_lock lock(localSubscribeMapLock_); - for (auto &item : unFinishedLocalAutoSubMap_) { - if (item.second.size() == 0) { - continue; - } - allSyncQueries[item.first] = {}; - auto iter = allSyncQueries.find(item.first); - for (const auto &queryId : item.second) { - auto iterTmp = localSubscribeTotalMap_.find(queryId); - if (iterTmp == localSubscribeTotalMap_.end()) { - LOGI("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryId)); - continue; - } - iter->second.push_back(iterTmp->second.first); - } - } -} - -void SingleVerSubscribeManager::RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - RemoveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - RemoveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); - if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && - unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { - unFinishedLocalAutoSubMap_[device].erase(queryId); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s delete from UnFinishedMap", - STR_MASK(device), STR_MASK(queryId)); - if (unFinishedLocalAutoSubMap_[device].size() == 0) { - LOGI("[SingleVerSubscribeManager] dev=%s delete from unFinish map", STR_MASK(device)); - unFinishedLocalAutoSubMap_.erase(device); - } - } -} - -void SingleVerSubscribeManager::GetLocalSubscribeQueries(const std::string &device, - std::vector &subscribeQueries) const -{ - std::shared_lock lock(localSubscribeMapLock_); - GetSubscribeQueries(device, localSubscribeMap_, localSubscribeTotalMap_, subscribeQueries); -} - -void SingleVerSubscribeManager::GetRemoteSubscribeQueries(const std::string &device, - std::vector &subscribeQueries) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - GetSubscribeQueries(device, remoteSubscribedMap_, remoteSubscribedTotalMap_, subscribeQueries); -} - -bool SingleVerSubscribeManager::IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - auto iter = remoteSubscribedMap_.find(device); - if (iter == remoteSubscribedMap_.end()) { - LOGD("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); - return false; - } - std::string queryId = query.GetIdentify(); - auto subIter = iter->second.find(queryId); - if (subIter == iter->second.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryId)); - return false; - } - return true; -} - -void SingleVerSubscribeManager::GetRemoteSubscribeQueryIds(const std::string &device, - std::vector &subscribeQueryIds) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - auto iter = remoteSubscribedMap_.find(device); - if (iter == remoteSubscribedMap_.end()) { - LOGI("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : iter->second) { - if (remoteSubscribedTotalMap_.find(queryInfo.first) == remoteSubscribedTotalMap_.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryInfo.first)); - continue; - } - subscribeQueryIds.push_back(queryInfo.first); - } -} - -int SingleVerSubscribeManager::LocalSubscribeLimitCheck(const std::vector &devices, - QuerySyncObject &query) const -{ - std::shared_lock lock(localSubscribeMapLock_); - int devNum = localSubscribeMap_.size(); - for (const auto &device : devices) { - if (localSubscribeMap_.find(device) != localSubscribeMap_.end()) { - continue; - } - devNum++; - if (devNum > MAX_DEVICES_NUM) { - LOGE("[SingleVerSubscribeManager] local subscribe devices is over limit"); - return -E_MAX_LIMITS; - } - } - std::string queryId = query.GetIdentify(); - auto allIter = localSubscribeTotalMap_.find(queryId); - if (allIter == localSubscribeTotalMap_.end() && localSubscribeTotalMap_.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { - LOGE("[SingleVerSubscribeManager] all local subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - return E_OK; -} - -void SingleVerSubscribeManager::ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGI("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : subscribeMap[device]) { - if (subscribedTotalMap.find(queryInfo.first) != subscribedTotalMap.end()) { - if (subscribedTotalMap[queryInfo.first].second > 0) { - subscribedTotalMap[queryInfo.first].second--; - } - if (subscribedTotalMap[queryInfo.first].second == 0) { - LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryInfo.first)); - subscribedTotalMap.erase(queryInfo.first); - } - } - } - subscribeMap.erase(device); - LOGI("[SingleVerSubscribeManager] clear dev=%s` remote subscribe queies finished", STR_MASK(device)); -} - -int SingleVerSubscribeManager::ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - std::string queryId = query.GetIdentify(); - auto iter = subscribeMap.find(device); - auto allIter = subscribedTotalMap.find(queryId); - // limit check - if (allIter == subscribedTotalMap.end() && subscribedTotalMap.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { - LOGE("[SingleVerSubscribeManager] all subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - if (iter == subscribeMap.end() && subscribeMap.size() >= MAX_DEVICES_NUM) { - LOGE("[SingleVerSubscribeManager] subscribe devices is over limit"); - return -E_MAX_LIMITS; - } - if (iter != subscribeMap.end() && iter->second.find(queryId) == iter->second.end() && - iter->second.size() >= MAX_SUBSCRIBE_NUM_PER_DEV) { - LOGE("[SingleVerSubscribeManager] subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - if (iter != subscribeMap.end() && iter->second.find(queryId) != iter->second.end() && - iter->second[queryId] == SubscribeStatus::ACTIVE) { - LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s already active in map", - STR_MASK(device), STR_MASK(queryId)); - return E_OK; - } - - if (iter == subscribeMap.end()) { - subscribeMap[device] = std::map {}; - } - bool isNeedInc = false; - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - subscribeMap[device][queryId] = SubscribeStatus::NOT_ACTIVE; - isNeedInc = true; - } - if (allIter == subscribedTotalMap.end()) { - subscribedTotalMap[queryId] = {query, 1}; - } else if (isNeedInc) { - subscribedTotalMap[queryId].second++; - } - return E_OK; -} - -int SingleVerSubscribeManager::ActiveSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribedTotalMap.find(queryId) == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); - return -E_INTERNAL_ERROR; - } - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s in localSubscribeMap", STR_MASK(device)); - return -E_INTERNAL_ERROR; - } - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); - return -E_INTERNAL_ERROR; - } - subscribeMap[device][queryId] = SubscribeStatus::ACTIVE; - return E_OK; -} - -void SingleVerSubscribeManager::DeleteSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s in map", STR_MASK(device)); - return; - } - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); - return; - } - SubscribeStatus queryStatus = subscribeMap[device][queryId]; - // not permit to delete the query when something wrong this time,because it is subscribed successfully last time - if (queryStatus == SubscribeStatus::ACTIVE) { - LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s is active, no need to del", - STR_MASK(device), STR_MASK(queryId)); - return; - } - subscribeMap[device].erase(queryId); - auto iter = subscribedTotalMap.find(queryId); - if (iter == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); - return; - } - iter->second.second--; - if (iter->second.second <= 0) { - LOGI("[SingleVerSubscribeManager] del queryId=%s from SubscribeTotalMap", STR_MASK(queryId)); - subscribedTotalMap.erase(queryId); - } - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribeMap success", - STR_MASK(device), STR_MASK(queryId)); -} - -void SingleVerSubscribeManager::RemoveSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - auto iter = subscribeMap.find(device); - if (iter == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); - return; - } - if (iter->second.find(queryId) == subscribeMap[device].end()) { - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s not in SubscribedMap", STR_MASK(device), STR_MASK(queryId)); - return; - } - iter->second.erase(queryId); - auto allIter = subscribedTotalMap.find(queryId); - if (allIter == subscribedTotalMap.end()) { - LOGI("[SingleVerSubscribeManager] queryId=%s not in TotalMap", STR_MASK(queryId)); - return; - } - allIter->second.second--; - if (allIter->second.second <= 0) { - subscribedTotalMap.erase(queryId); - LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryId)); - } - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribedMap success", - STR_MASK(device), STR_MASK(queryId)); -} - -void SingleVerSubscribeManager::GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, - const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const -{ - auto iter = subscribeMap.find(device); - if (iter == subscribeMap.end()) { - LOGD("[SingleVerSubscribeManager] dev=%s not in localSubscribeMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : iter->second) { - auto iterTmp = subscribedTotalMap.find(queryInfo.first); - if (iterTmp == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryInfo.first)); - continue; - } - subscribeQueries.push_back(iterTmp->second.first); - } -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h deleted file mode 100644 index c152e0f29e0f966436f3e8c2d048add4aa3e6f2c..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SINGLE_VER_SUBSCRIBE_MANAGER_H -#define SINGLE_VER_SUBSCRIBE_MANAGER_H - -#include -#include -#include "query_sync_object.h" - -namespace DistributedDB { -enum class SubscribeStatus { - NOT_ACTIVE = 0, - ACTIVE = 1, -}; - -using SubscribeMap = std::map>; -using SubscribedTotalMap = std::map>; - -class SingleVerSubscribeManager { -public: - SingleVerSubscribeManager() = default; - ~SingleVerSubscribeManager() {}; - - DISABLE_COPY_ASSIGN_MOVE(SingleVerSubscribeManager); - - // clear remoteSubscribeMap_[device] list when remote db is closed or dev offline. - void ClearRemoteSubscribeQuery(const std::string &device); - - // clear localSubscribeMap_[device] list when device is offline. - void ClearLocalSubscribeQuery(const std::string &device); - - void ClearAllRemoteQuery(); - - // add query when receive subscribe command - int ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // active query to ACTIVE when send ack ok - int ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // reserve query when user call SubscribeRemoteQuery, status set to NOT_ACTIVE - int ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // active query to ACTIVE when receive ack ok - int ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // delete local subscribe query when recv wrong errCode, only not_active status allowed to del - void DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // delete remote subscribe query when send msg failed, only not_active status allowed to del - void DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // put subscribe queries into unfinished map when remote db online - void PutLocalUnFiniedSubQueries(const std::string &device, std::vector &subscribeQueries); - - // get all device unFinished subscribe queries which triggered by auto subscribe and need retry subscribe - void GetAllUnFinishSubQueries(std::map> &allSyncQueries) const; - - // remove query when receive unsubscribe command - void RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // remove query when user call UnSubscribeRemoteQuery - void RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // get device active subscribeQueries from localSubscribeMap_ - void GetLocalSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; - - // get device remote queryId from remoteSubscribedMap_ while data change - void GetRemoteSubscribeQueryIds(const std::string &device, std::vector &subscribeQueryIds) const; - // get device remote subscribeQueries from remoteSubscribedMap_ while data change - void GetRemoteSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; - - bool IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const; - - int LocalSubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const; -private: - void ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - int ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - int ActiveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void DeleteSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void RemoveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, - const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const; - - mutable std::shared_mutex localSubscribeMapLock_; - // subscribe sponsor, key: device, value: pair map - // status 0: active, 1: not active - SubscribeMap localSubscribeMap_; - - // used retry subscribe in db open scene, key: device value: set - std::map> unFinishedLocalAutoSubMap_; - - // subscribe sponsor total query info, key:queryId, value: - // while use_num is 0, delete item from the map - SubscribedTotalMap localSubscribeTotalMap_; - - mutable std::shared_mutex remoteSubscribedMapLock_; - // subscribed, key: device, value: pair map - // status 0: active, 1: not active - SubscribeMap remoteSubscribedMap_; - - // subscribed total query info, key:queryId, value: - // while use_num is 0, delete item from the map - SubscribedTotalMap remoteSubscribedTotalMap_; -}; -} // namespace DistributedDB - -#endif // SINGLE_VER_SUBSCRIBE_MANAGER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp index 0dc5fe0007fddcf822c3fdc0157e10054a83036e..a59713da0dd6a324b45695cdbaccfe60e8231669 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp @@ -37,8 +37,9 @@ namespace { const int EVENT_INDEX = 1; const int OUTPUT_STATE_INDEX = 2; - // State switch table v1 and v2, has three columns, CurrentState, Event, and OutSate - const std::vector> STATE_SWITCH_TABLE_V2 = { + // drop v1 and v2 table by one optimize, dataSend mode in all version go with slide window mode. + // State switch table v3, has three columns, CurrentState, Event, and OutSate + const std::vector> STATE_SWITCH_TABLE_V3 = { {IDLE, START_SYNC_EVENT, TIME_SYNC}, // In TIME_SYNC state @@ -48,45 +49,25 @@ namespace { // In ABILITY_SYNC state, compare version num and schema {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, - {ABILITY_SYNC, SWITCH_TO_PROCTOL_V1_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, {ABILITY_SYNC, CONTROL_CMD_EVENT, SYNC_CONTROL_CMD}, // In START_INITIACTIVE_DATA_SYNC state, send a sync request, and send first packt of data sync - {START_INITIACTIVE_DATA_SYNC, SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, - {START_INITIACTIVE_DATA_SYNC, RESPONSE_PUSH_REMAINDER_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_INITIACTIVE_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, {START_INITIACTIVE_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {START_INITIACTIVE_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_INITIACTIVE_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_DATA_SYNC}, {START_INITIACTIVE_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, - - // In INACTIVE_PUSH_REMAINDER_DATA state, do initiactive sync, send remainder pcket of data sync - {INACTIVE_PUSH_REMAINDER_DATA, SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, - {INACTIVE_PUSH_REMAINDER_DATA, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, - {INACTIVE_PUSH_REMAINDER_DATA, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {INACTIVE_PUSH_REMAINDER_DATA, INNER_ERR_EVENT, INNER_ERR}, - {INACTIVE_PUSH_REMAINDER_DATA, RE_SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, - {INACTIVE_PUSH_REMAINDER_DATA, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + {START_INITIACTIVE_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_DATA_SYNC}, // In START_PASSIVE_DATA_SYNC state, do response pull request, and send first packt of data sync - {START_PASSIVE_DATA_SYNC, SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_PASSIVE_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, {START_PASSIVE_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, {START_PASSIVE_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {START_PASSIVE_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_PASSIVE_DATA_SYNC, RE_SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_PASSIVE_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - - // In PASSIVE_PUSH_REMAINDER_DATA state, do passive sync, send remainder pcket of data sync - {PASSIVE_PUSH_REMAINDER_DATA, SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, - {PASSIVE_PUSH_REMAINDER_DATA, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, - {PASSIVE_PUSH_REMAINDER_DATA, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {PASSIVE_PUSH_REMAINDER_DATA, INNER_ERR_EVENT, INNER_ERR}, - {PASSIVE_PUSH_REMAINDER_DATA, RE_SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, - {PASSIVE_PUSH_REMAINDER_DATA, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + {START_PASSIVE_DATA_SYNC, RE_SEND_DATA_EVENT, START_PASSIVE_DATA_SYNC}, // In WAIT_FOR_RECEIVE_DATA_FINISH, {WAIT_FOR_RECEIVE_DATA_FINISH, RECV_FINISHED_EVENT, SYNC_TASK_FINISHED}, @@ -108,58 +89,6 @@ namespace { {SYNC_TIME_OUT, ANY_EVENT, SYNC_TASK_FINISHED}, {INNER_ERR, ANY_EVENT, SYNC_TASK_FINISHED}, }; - - // State switch table v3, has three columns, CurrentState, Event, and OutSate - const std::vector> STATE_SWITCH_TABLE_V3 = { - {IDLE, START_SYNC_EVENT, TIME_SYNC}, - - // In TIME_SYNC state - {TIME_SYNC, TIME_SYNC_FINISHED_EVENT, ABILITY_SYNC}, - {TIME_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {TIME_SYNC, INNER_ERR_EVENT, INNER_ERR}, - - // In ABILITY_SYNC state, compare version num and schema - {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, - {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, - {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {ABILITY_SYNC, CONTROL_CMD_EVENT, SYNC_CONTROL_CMD}, - - // In START_INITIACTIVE_SLIDING_DATA_SYNC state, send a sync request, and send first packt of data sync - {START_INITIACTIVE_SLIDING_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, - - // In START_PASSIVE_SLIDING_DATA_SYNC state, do response pull request, and send first packt of data sync - {START_PASSIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - {START_PASSIVE_SLIDING_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, - {START_PASSIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {START_PASSIVE_SLIDING_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_PASSIVE_SLIDING_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - {START_PASSIVE_SLIDING_DATA_SYNC, RE_SEND_DATA_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - - // In WAIT_FOR_RECEIVE_DATA_FINISH, - {WAIT_FOR_RECEIVE_DATA_FINISH, RECV_FINISHED_EVENT, SYNC_TASK_FINISHED}, - {WAIT_FOR_RECEIVE_DATA_FINISH, START_PULL_RESPONSE_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - {WAIT_FOR_RECEIVE_DATA_FINISH, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {WAIT_FOR_RECEIVE_DATA_FINISH, INNER_ERR_EVENT, INNER_ERR}, - {WAIT_FOR_RECEIVE_DATA_FINISH, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - - {SYNC_CONTROL_CMD, SEND_FINISHED_EVENT, SYNC_TASK_FINISHED}, - {SYNC_CONTROL_CMD, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {SYNC_CONTROL_CMD, INNER_ERR_EVENT, INNER_ERR}, - {SYNC_CONTROL_CMD, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - - // In SYNC_TASK_FINISHED, - {SYNC_TASK_FINISHED, ALL_TASK_FINISHED_EVENT, IDLE}, - {SYNC_TASK_FINISHED, START_SYNC_EVENT, TIME_SYNC}, - - // SYNC_TIME_OUT and INNE_ERR state, just do some exception resolve - {SYNC_TIME_OUT, ANY_EVENT, SYNC_TASK_FINISHED}, - {INNER_ERR, ANY_EVENT, SYNC_TASK_FINISHED}, - }; } std::mutex SingleVerSyncStateMachine::stateSwitchTableLock_; @@ -197,7 +126,6 @@ int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, ISyncInterf timeSync_ = std::make_unique(); dataSync_ = std::make_shared(); abilitySync_ = std::make_unique(); - dataSyncWithSlidingWindow_ = std::make_unique(); errCode = timeSync_->Initialize(communicator, metaData, syncInterface, context->GetDeviceId()); if (errCode != E_OK) { @@ -215,14 +143,7 @@ int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, ISyncInterf currentState_ = IDLE; context_ = static_cast(context); syncInterface_ = static_cast(syncInterface); - errCode = dataSyncWithSlidingWindow_->ReceiverInit(context_, dataSync_); - if (errCode != E_OK) { - goto ERROR_OUT; - } - errCode = dataSyncWithSlidingWindow_->SenderInit(context_, dataSync_); - if (errCode != E_OK) { - goto ERROR_OUT; - } + InitStateSwitchTables(); InitStateMapping(); return E_OK; @@ -324,11 +245,10 @@ void SingleVerSyncStateMachine::AbortInner() { LOGE("[StateMachine][AbortInner] error occurred,abort,label=%s,dev=%s", dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - int mode = SyncOperation::TransferSyncMode(context_->GetMode()); - if (mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL || context_->IsKilled()) { - dataSyncWithSlidingWindow_->ReceiverClear(); + if (context_->IsKilled()) { + dataSync_->ClearDataMsg(); } - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); ContinueToken token; context_->GetContinueToken(token); if (token != nullptr) { @@ -384,7 +304,6 @@ void SingleVerSyncStateMachine::InitStateSwitchTables() return; } - InitStateSwitchTable(SINGLE_VER_SYNC_PROCTOL_V2, STATE_SWITCH_TABLE_V2); InitStateSwitchTable(SINGLE_VER_SYNC_PROCTOL_V3, STATE_SWITCH_TABLE_V3); std::sort(stateSwitchTables_.begin(), stateSwitchTables_.end(), [](const auto &tableA, const auto &tableB) { @@ -419,54 +338,17 @@ void SingleVerSyncStateMachine::InitStateMapping() { stateMapping_[TIME_SYNC] = std::bind(&SingleVerSyncStateMachine::DoTimeSync, this); stateMapping_[ABILITY_SYNC] = std::bind(&SingleVerSyncStateMachine::DoAbilitySync, this); - stateMapping_[START_INITIACTIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveDataSync, - this); - stateMapping_[START_PASSIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoPassiveDataSync, this); - stateMapping_[INACTIVE_PUSH_REMAINDER_DATA] = std::bind(&SingleVerSyncStateMachine::DoInitiactivePushRemainderData, - this); - stateMapping_[PASSIVE_PUSH_REMAINDER_DATA] = std::bind(&SingleVerSyncStateMachine::DoPassivePushRemainderData, - this); stateMapping_[WAIT_FOR_RECEIVE_DATA_FINISH] = std::bind(&SingleVerSyncStateMachine::DoWaitForDataRecv, this); stateMapping_[SYNC_TASK_FINISHED] = std::bind(&SingleVerSyncStateMachine::DoSyncTaskFinished, this); stateMapping_[SYNC_TIME_OUT] = std::bind(&SingleVerSyncStateMachine::DoTimeout, this); stateMapping_[INNER_ERR] = std::bind(&SingleVerSyncStateMachine::DoInnerErr, this); - stateMapping_[START_INITIACTIVE_SLIDING_DATA_SYNC] = + stateMapping_[START_INITIACTIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow, this); - stateMapping_[START_PASSIVE_SLIDING_DATA_SYNC] = + stateMapping_[START_PASSIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow, this); stateMapping_[SYNC_CONTROL_CMD] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveControlSync, this); } -Event SingleVerSyncStateMachine::DoInitiactiveDataSync() -{ - int errCode = E_OK; - int mode = SyncOperation::TransferSyncMode(context_->GetMode()); - switch (mode) { - case SyncModeType::PUSH: - context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); - errCode = dataSync_->PushStart(context_); - break; - case SyncModeType::PULL: - context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - errCode = dataSync_->PullRequestStart(context_); - break; - case SyncModeType::PUSH_AND_PULL: - errCode = dataSync_->PushPullStart(context_); - break; - case SyncModeType::RESPONSE_PULL: - // In response pull mode, reminader data should send in - // PASSIVE_PUSH_REMAINDER_DATA - return Event::RESPONSE_PUSH_REMAINDER_EVENT; - default: - errCode = -E_NOT_SUPPORT; - break; - } - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() { LOGD("[StateMachine][activeDataSync] mode=%d,label=%s,dev=%s", context_->GetMode(), @@ -476,19 +358,19 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() case SyncModeType::PUSH: case SyncModeType::QUERY_PUSH: context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; case SyncModeType::PULL: case SyncModeType::QUERY_PULL: context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; case SyncModeType::PUSH_AND_PULL: case SyncModeType::QUERY_PUSH_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; case SyncModeType::RESPONSE_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; default: errCode = -E_NOT_SUPPORT; @@ -509,23 +391,6 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() return (ignoreInnerErr && event == INNER_ERR_EVENT) ? SEND_FINISHED_EVENT : event; } -Event SingleVerSyncStateMachine::DoPassiveDataSync() -{ - { - RefObject::AutoLock lock(context_); - if (context_->GetRspTargetQueueSize() != 0) { - PreStartPullResponse(); - } else { - return RESPONSE_TASK_FINISHED_EVENT; - } - } - int errCode = dataSync_->PullResponseStart(context_); - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - Event SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow() { { @@ -536,7 +401,7 @@ Event SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow() return RESPONSE_TASK_FINISHED_EVENT; } } - int errCode = dataSyncWithSlidingWindow_->SenderStart(SyncModeType::RESPONSE_PULL, context_, dataSync_); + int errCode = dataSync_->SyncStart(SyncModeType::RESPONSE_PULL, context_); if (errCode != E_OK) { LOGW("[SingleVerSyncStateMachine][DoPassiveDataSyncWithSlidingWindow] response pull send failed[%d]", errCode); return RESPONSE_TASK_FINISHED_EVENT; @@ -569,39 +434,6 @@ int SingleVerSyncStateMachine::HandleControlAckRecv(const Message *inMsg) return E_OK; } -Event SingleVerSyncStateMachine::DoInitiactivePushRemainderData() -{ - int errCode; - int mode = SyncOperation::TransferSyncMode(context_->GetMode()); - switch (mode) { - case SyncModeType::PULL: - case SyncModeType::RESPONSE_PULL: - // In pull or response pul mode, don't need to do INACTIVE_PUSH - return Event::SEND_FINISHED_EVENT; - case SyncModeType::PUSH: - case SyncModeType::PUSH_AND_PULL: - errCode = dataSync_->PushStart(context_); - break; - default: - errCode = -E_INTERNAL_ERROR; - break; - } - - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - -Event SingleVerSyncStateMachine::DoPassivePushRemainderData() -{ - int errCode = dataSync_->PullResponseStart(context_); - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - Event SingleVerSyncStateMachine::DoWaitForDataRecv() const { if (context_->GetRspTargetQueueSize() != 0) { @@ -676,17 +508,13 @@ Event SingleVerSyncStateMachine::GetEventAfterTimeSync(int mode, bool isEarliest if (mode == SyncModeType::SUBSCRIBE_QUERY || mode == SyncModeType::UNSUBSCRIBE_QUERY) { return Event::CONTROL_CMD_EVENT; } - if (isEarliestVersion) { - LOGI("remote version is 0, switch to v1 proctol"); - return Event::SWITCH_TO_PROCTOL_V1_EVENT; - } return Event::ABILITY_SYNC_FINISHED_EVENT; } Event SingleVerSyncStateMachine::DoSyncTaskFinished() { StopWatchDog(); - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); RefObject::AutoLock lock(syncContext_); int errCode = ExecNextTask(); if (errCode == E_OK) { @@ -786,6 +614,15 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) { + TimeOffset offset = 0; + uint32_t timeout = communicator_->GetTimeout(context_->GetDeviceId()); + // If message is data sync request, we should check timeoffset. + int errCode = timeSync_->GetTimeOffset(offset, timeout); + if (errCode != E_OK) { + LOGE("[StateMachine][HandleDataRequestRecv] GetTimeOffset err! errCode=%d", errCode); + return errCode; + } + context_->SetTimeOffset(offset); PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); if (performance != nullptr) { performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); @@ -802,7 +639,7 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) // So we need to send save data notify to keep remote alive. bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); WaterMark pullEndWaterkark = 0; - int errCode = dataSync_->DataRequestRecv(context_, inMsg, pullEndWaterkark); + errCode = dataSync_->DataRequestRecv(context_, inMsg, pullEndWaterkark); if (performance != nullptr) { performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); } @@ -823,22 +660,14 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) return E_OK; } -int SingleVerSyncStateMachine::PreHandleAckRecv(const Message *inMsg) -{ - if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - return dataSyncWithSlidingWindow_->PreHandleSenderAckRecv(inMsg); - } - return E_OK; -} - void SingleVerSyncStateMachine::HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg, bool ignoreInnerErr) { if (errCode == -E_RE_SEND_DATA) { // LOCAL_WATER_MARK_NOT_INIT - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); } if (errCode == -E_NO_DATA_SEND || errCode == -E_SEND_DATA) { - int ret = dataSyncWithSlidingWindow_->SenderAckRecv(inMsg); + int ret = dataSync_->TryContinueSync(context_, inMsg); if (ret == -E_FINISHED) { SwitchStateAndStep(Event::SEND_FINISHED_EVENT); return; @@ -862,7 +691,7 @@ void SingleVerSyncStateMachine::NeedAbilitySyncHandle() currentRemoteVersionId_ = context_->GetRemoteSoftwareVersionId(); } abilitySync_->SetAbilitySyncFinishedStatus(false); - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); } int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) @@ -881,22 +710,16 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) if (IsNeedResetWatchdog(inMsg)) { (void)ResetWatchDog(); } - int errCode = PreHandleAckRecv(inMsg); - if (errCode != E_OK) { + if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && !dataSync_->AckPacketIdCheck(inMsg)) { // packetId not match but sequence id matched scene, means resend map has be rebuilt // this is old ack, shoulb be dropped and wait for the same packetId sequence. - if (errCode == -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG) { - return E_OK; - } - // this means error happened,should stop the sync task. - SwitchStateAndStep(TransformErrCodeToEvent(errCode)); - return errCode; + return E_OK; } // AckRecv will save meta data, it may cost a long time. if another thread is saving data // So we need to send save data notify to keep remote alive. // eg. remote do pull sync bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); - errCode = dataSync_->AckRecv(context_, inMsg); + int errCode = dataSync_->AckRecv(context_, inMsg); if (isNeedStop) { StopSaveDataNotify(); } @@ -909,10 +732,6 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) bool ignoreInnerErr = inMsg->GetSessionId() == context_->GetResponseSessionId() && context_->GetRequestSessionId() != 0; DataAckRecvErrCodeHandle(errCode, !ignoreInnerErr); - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - ResponsePullError(errCode, ignoreInnerErr); - return errCode; - } HandleDataAckRecvWithSlidingWindow(errCode, inMsg, ignoreInnerErr); return errCode; } @@ -920,26 +739,11 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) int SingleVerSyncStateMachine::DataPktRecv(Message *inMsg) { PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - int errCode; - TimeOffset offset = 0; - uint32_t timeout = TIME_SYNC_WAIT_TIME; - + int errCode = E_OK; switch (inMsg->GetMessageType()) { case TYPE_REQUEST: - timeout = communicator_->GetTimeout(context_->GetDeviceId()); - // If message is data sync request, we should check timeoffset. - errCode = timeSync_->GetTimeOffset(offset, timeout); - if (errCode != E_OK) { - LOGE("[StateMachine][DataPktRecv] GetTimeOffset err! errCode=%d", errCode); - return errCode; - } - context_->SetTimeOffset(offset); - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - // higher than 102 version may go to here, but it is ok not use slwr,after abilitysync would go slwr. - errCode = HandleDataRequestRecv(inMsg); - } else { - errCode = dataSyncWithSlidingWindow_->Receive(inMsg); - } + ScheduleMsgAndHandle(inMsg); + errCode = -E_NOT_NEED_DELETE_MSG; break; case TYPE_RESPONSE: case TYPE_NOTIFY: @@ -956,6 +760,43 @@ int SingleVerSyncStateMachine::DataPktRecv(Message *inMsg) return errCode; } +void SingleVerSyncStateMachine::ScheduleMsgAndHandle(Message *inMsg) +{ + dataSync_->PutDataMsg(inMsg); + while (true) { + bool isNeedHandle = true; + bool isNeedContinue = true; + Message *msg = dataSync_->MoveNextDataMsg(context_, isNeedHandle, isNeedContinue); + if (!isNeedContinue) { + break; + } + if (msg == nullptr) { + if (dataSync_->IsNeedReloadQueue()) { + continue; + } + break; + } + bool isNeedClearMap = false; + if (isNeedHandle) { + int errCode = HandleDataRequestRecv(msg); + if (context_->IsReceiveWaterMarkErr() || errCode == -E_NEED_ABILITY_SYNC) { + isNeedClearMap = true; + } + if (errCode == -E_TIMEOUT) { + isNeedHandle = false; + } + } else { + dataSync_->SendFinishedDataAck(context_, msg); + } + if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { + // for lower version, no need to handle map schedule info, just reset schedule working status + isNeedHandle = false; + } + dataSync_->ScheduleInfoHandle(isNeedHandle, isNeedClearMap, msg); + delete msg; + } +} + int SingleVerSyncStateMachine::ControlPktRecv(Message *inMsg) { int errCode = E_OK; @@ -1034,7 +875,6 @@ int SingleVerSyncStateMachine::TimeMarkSyncRecv(const Message *inMsg) void SingleVerSyncStateMachine::Clear() { - dataSyncWithSlidingWindow_ = nullptr; dataSync_ = nullptr; timeSync_ = nullptr; abilitySync_ = nullptr; @@ -1073,18 +913,7 @@ bool SingleVerSyncStateMachine::IsPacketValid(const Message *inMsg) const inMsg->GetMessageId() == CONTROL_SYNC_MESSAGE) { return true; } - - // sliding window don't need to check sequenceId here, just return - if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - return true; - } - - if (isResponseType && (inMsg->GetSequenceId() != context_->GetSequenceId())) { - LOGE("[StateMachine][IsPacketValid] Message is invalid,inMsg SequenceId=%d,seqId=%d,syncId=%d", - inMsg->GetSequenceId(), context_->GetSequenceId(), context_->GetSyncId()); - return false; - } - + // sequenceId will be checked in dataSync return true; } @@ -1100,23 +929,6 @@ void SingleVerSyncStateMachine::PreStartPullResponse() context_->SetQuery(target.GetQuery()); } -bool SingleVerSyncStateMachine::IsRightDataResponsePkt(const Message *inMsg) const -{ - if (inMsg->GetMessageId() != DATA_SYNC_MESSAGE || inMsg->GetMessageId() != QUERY_SYNC_MESSAGE) { - return false; - } - - if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - return false; - } - - if ((context_->GetMode() != SyncModeType::INVALID_MODE) && (inMsg->GetMessageType() == TYPE_RESPONSE)) { - return true; - } - - return false; -} - bool SingleVerSyncStateMachine::CheckIsStartPullResponse() const { // Other state will step to do pull response, only this statem we need to step the statemachine @@ -1136,10 +948,6 @@ int SingleVerSyncStateMachine::MessageCallbackPre(const Message *inMsg) if (!IsPacketValid(inMsg)) { return -E_INVALID_ARGS; } - - if (IsRightDataResponsePkt(inMsg)) { - context_->IncSequenceId(); - } return E_OK; } @@ -1238,11 +1046,6 @@ Event SingleVerSyncStateMachine::TransforTimeOutErrCodeToEvent() } } -void SingleVerSyncStateMachine::SetSlidingWindowSenderErr(bool isErr) -{ - dataSyncWithSlidingWindow_->SetSenderErr(isErr); -} - bool SingleVerSyncStateMachine::IsNeedErrCodeHandle(uint32_t sessionId) const { // omit to set sessionId so version_3 should skip to compare sessionid. @@ -1320,12 +1123,6 @@ void SingleVerSyncStateMachine::DataAckRecvErrCodeHandle(int errCode, bool handl case -E_NEED_ABILITY_SYNC: NeedAbilitySyncHandle(); break; - case -E_NO_DATA_SEND: - // when version is higher than 102, this step will be handle in slide windows function. - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0 && handleError) { - context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - } - break; case -E_NOT_PERMIT: if (handleError) { context_->SetOperationStatus(SyncOperation::OP_PERMISSION_CHECK_FAILED); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h index 5fbb258f12703e76440453fed7007a0254b6622d..965c13eebaf8fec07b65ec7b4f4d6f279cdd8b8b 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h @@ -24,7 +24,6 @@ #include "meta_data.h" #include "semaphore_utils.h" #include "single_ver_data_sync.h" -#include "single_ver_data_sync_with_sliding_window.h" #include "single_ver_sync_task_context.h" #include "sync_state_machine.h" #include "sync_target.h" @@ -38,16 +37,12 @@ namespace { IDLE = 0, TIME_SYNC, ABILITY_SYNC, - START_INITIACTIVE_DATA_SYNC, // used to do sync started by local device - START_PASSIVE_DATA_SYNC, // used to do sync response remote device - INACTIVE_PUSH_REMAINDER_DATA, // push remainded data if initactive sync data more than on frame - PASSIVE_PUSH_REMAINDER_DATA, // push remainded data if passive sync data more than on frame WAIT_FOR_RECEIVE_DATA_FINISH, // all data send finished, wait for data revice if has pull request SYNC_TASK_FINISHED, // current sync task finihsed, try to schedule next sync task SYNC_TIME_OUT, INNER_ERR, - START_INITIACTIVE_SLIDING_DATA_SYNC, // used to do sync started by local device, use sliding window - START_PASSIVE_SLIDING_DATA_SYNC, // used to do pull response, use sliding window + START_INITIACTIVE_DATA_SYNC, // used to do sync started by local device, use sliding window + START_PASSIVE_DATA_SYNC, // used to do pull response, use sliding window SYNC_CONTROL_CMD // used to send control cmd. }; @@ -56,12 +51,10 @@ namespace { TIME_SYNC_FINISHED_EVENT, ABILITY_SYNC_FINISHED_EVENT, VERSION_NOT_SUPPOR_EVENT, - SWITCH_TO_PROCTOL_V1_EVENT, SEND_DATA_EVENT, SEND_FINISHED_EVENT, RECV_FINISHED_EVENT, NEED_ABILITY_SYNC_EVENT, - RESPONSE_PUSH_REMAINDER_EVENT, RESPONSE_TASK_FINISHED_EVENT, START_PULL_RESPONSE_EVENT, WAIT_ACK_EVENT, @@ -94,8 +87,6 @@ public: int HandleDataRequestRecv(const Message *inMsg); - void SetSlidingWindowSenderErr(bool isErr); - bool IsNeedErrCodeHandle(uint32_t sessionId) const; void PushPullDataRequestEvokeErrHandle(); @@ -151,18 +142,6 @@ private: // Do AbilitySync, for first sync Event DoAbilitySync(); - // Do data packet sync for initiactive sync operation, if need pull, there need send a pull request. - Event DoInitiactiveDataSync(); - - // Do data packet sync for response pull request. - Event DoPassiveDataSync(); - - // Push remainder data to remote - Event DoInitiactivePushRemainderData(); - - // Push remainder data to remote for response pull request - Event DoPassivePushRemainderData(); - // Waiting for pull data revice finish, if coming a pull request, should goto START_PASSIVE_DATA_SYNC state Event DoWaitForDataRecv() const; @@ -193,14 +172,14 @@ private: int DataPktRecv(Message *inMsg); + void ScheduleMsgAndHandle(Message *inMsg); + int ControlPktRecv(Message *inMsg); void NeedAbilitySyncHandle(); int HandleDataAckRecv(const Message *inMsg); - int PreHandleAckRecv(const Message *inMsg); - void HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg, bool ignoreInnerErr); void ResponsePullError(int errCode, bool ignoreInnerErr); @@ -211,8 +190,6 @@ private: void PreStartPullResponse(); - bool IsRightDataResponsePkt(const Message *inMsg) const; - bool CheckIsStartPullResponse() const; int MessageCallbackPre(const Message *inMsg); @@ -245,7 +222,6 @@ private: std::unique_ptr timeSync_; std::unique_ptr abilitySync_; std::shared_ptr dataSync_; - std::unique_ptr dataSyncWithSlidingWindow_; uint64_t currentRemoteVersionId_; std::map stateMapping_; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp index d07b32898fb788b571d5e6b9138141dab3f6e8db..cbc1fbb563433f716f8da2d65d8399378f7163f3 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp @@ -307,27 +307,6 @@ void SingleVerSyncTaskContext::StopFeedDogForSync(SyncDirectionFlag flag) stateMachine_->StopFeedDogForSync(flag); } -void SingleVerSyncTaskContext::SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange) -{ - dataTimeRange_ = dataTimeRange; -} - -SyncTimeRange SingleVerSyncTaskContext::GetDataTimeRange() const -{ - return dataTimeRange_; -} - - -void SingleVerSyncTaskContext::SetSessionEndTimeStamp(TimeStamp end) -{ - sessionEndTimeStamp_ = end; -} - -TimeStamp SingleVerSyncTaskContext::GetSessionEndTimeStamp() const -{ - return sessionEndTimeStamp_; -} - int SingleVerSyncTaskContext::HandleDataRequestRecv(const Message *msg) { return static_cast(stateMachine_)->HandleDataRequestRecv(msg); @@ -343,11 +322,6 @@ void SingleVerSyncTaskContext::SetReceiveWaterMarkErr(bool isErr) isReceiveWaterMarkErr_ = isErr; } -void SingleVerSyncTaskContext::SetSlidingWindowSenderErr(bool isErr) -{ - static_cast(stateMachine_)->SetSlidingWindowSenderErr(isErr); -} - void SingleVerSyncTaskContext::SetRemoteSeccurityOption(SecurityOption secOption) { remoteSecOption_ = secOption; @@ -546,7 +520,7 @@ bool SingleVerSyncTaskContext::IsCurrentSyncTaskCanBeSkipped() const return false; } if (localWaterMark > maxTimeStampInDb) { - LOGD("skip current push task, deviceId_ = %s, localWaterMark = %llu, maxTimeStampInDb = %llu", + LOGI("skip current push task, deviceId_ = %s, localWaterMark = %llu, maxTimeStampInDb = %llu", STR_MASK(deviceId_), localWaterMark, maxTimeStampInDb); return true; } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h index 221d65e9953ccf3a0c202230d1307eaf058bae12..85f29c2123aae99e81687629ddc2190f813b471d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h @@ -88,28 +88,14 @@ public: // stop timer to ResetWatchDog when sync data one (key,value) size bigger than mtu void StopFeedDogForSync(SyncDirectionFlag flag); - SyncTimeRange GetDataTimeRange() const; - virtual int HandleDataRequestRecv(const Message *msg); - // if sended by sliding window, set the start and and timeStamp of data in a sequence - void SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange); - - // if sended by sliding window, set the last data timeStamp in a sync session - void SetSessionEndTimeStamp(TimeStamp end); - - // if sended by sliding window, get the last data timeStamp in a sync session - TimeStamp GetSessionEndTimeStamp() const; - // is receive warterMark err bool IsReceiveWaterMarkErr() const; // set receive warterMark err void SetReceiveWaterMarkErr(bool isErr); - // set sliding window sender err - void SetSlidingWindowSenderErr(bool isErr); - void SetRemoteSeccurityOption(SecurityOption secOption); SecurityOption GetRemoteSeccurityOption() const; @@ -173,11 +159,6 @@ private: SyncStrategy syncStrategy_; bool isSchemaSync_ = false; - // normal data or delete data start timestamp and end timestamp,recorded for slws resend. - SyncTimeRange dataTimeRange_; - // in a sync session, the last data timeStamp - TimeStamp sessionEndTimeStamp_ = 0; - // is receive waterMark err, peerWaterMark bigger than remote localWaterMark bool isReceiveWaterMarkErr_ = false; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp deleted file mode 100755 index 3cb5c8a725d059d6120fcbdd68882b2c1e1b3679..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sliding_window_receiver.h" -#include "sync_task_context.h" -#include "db_common.h" - -namespace DistributedDB { -SlidingWindowReceiver::~SlidingWindowReceiver() -{ - Clear(); - dataSync_ = nullptr; - context_ = nullptr; -} - -void SlidingWindowReceiver::Clear() -{ - StopTimer(); - std::lock_guard lock(lock_); - ClearMap(); - isWaterMarkErrHappened_ = false; -} - -int SlidingWindowReceiver::Initialize(SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - if (context == nullptr || dataSync == nullptr) { - LOGE("[slwr] Initialize invalid args"); - return -E_INVALID_ARGS; - } - context_ = context; - dataSync_ = dataSync; - return E_OK; -} - -int SlidingWindowReceiver::Receive(Message *inMsg) -{ - LOGD("[slwr] receive msg"); - if (inMsg == nullptr) { - return -E_INVALID_ARGS; - } - int errCode = PutMsg(inMsg); - if (errCode == E_OK) { - StopTimer(); - StartTimer(); - DealMsg(); - return -E_NOT_NEED_DELETE_MSG; - } - return errCode; -} - -int SlidingWindowReceiver::PutMsg(Message *inMsg) -{ - const DataRequestPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - uint32_t sessionId = inMsg->GetSessionId(); - uint32_t sequenceId = inMsg->GetSequenceId(); - bool isLastSequence = packet->IsLastSequence(); - std::unique_lock lock(lock_); - if (workingId_ != 0 && sessionId_ != 0) { - LOGI("[PutMsg] task is running, wait for workdingId=%u end,seId=%u", workingId_, sequenceId); - workingTaskcv_.wait(lock, [this] { return workingId_ == 0; }); - } - if (sessionId_ != sessionId) { - ResetInfo(); - sessionId_ = sessionId; - messageMap_[sequenceId] = inMsg; - SetEndField(isLastSequence, sequenceId); - return E_OK; - } - // maybe remote has not receive ack, we resend ack. - if (sequenceId <= hasFinishedMaxId_) { - LOGI("[slwr] seId=%u,FinishedMId_=%u,label=%s", sequenceId, hasFinishedMaxId_, dataSync_->GetLabel().c_str()); - lock.unlock(); - dataSync_->SendDataAck(context_, inMsg, E_OK, 0); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - int errCode = ErrHandle(sequenceId, packet->GetPacketId()); - if (errCode != E_OK) { - return errCode; - } - if (messageMap_.count(sequenceId) > 0) { - LOGI("[slwr] PutMsg sequenceId already in map label=%s,deviceId=%s", - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - // it cannot be null here - const auto *cachePacket = messageMap_[sequenceId]->GetObject(); - if (cachePacket->GetPacketId() > packet->GetPacketId()) { - LOGI("[slwr] PutMsg receive old packet %u, new packet is %u", - packet->GetPacketId(), cachePacket->GetPacketId()); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - delete messageMap_[sequenceId]; - messageMap_[sequenceId] = nullptr; - } - messageMap_[sequenceId] = inMsg; - SetEndField(isLastSequence, sequenceId); - return E_OK; -} - -void SlidingWindowReceiver::DealMsg() -{ - while (true) { - Message *msg = nullptr; - uint64_t curPacketId = 0; - { - std::lock_guard lock(lock_); - if (workingId_ != 0 || messageMap_.count(hasFinishedMaxId_ + 1) == 0) { - LOGI("[slwr] DealMsg do nothing workingId_=%u,hasFinishedMaxId_=%u,label=%s,deviceId=%s", - workingId_, hasFinishedMaxId_, dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - return; - } - uint32_t curWorkingId = hasFinishedMaxId_ + 1; - msg = messageMap_[curWorkingId]; - messageMap_.erase(curWorkingId); - const auto *packet = msg->GetObject(); - curPacketId = packet->GetPacketId(); - if (curPacketId != 0 && curPacketId < hasFinishedPacketId_) { - delete msg; - msg = nullptr; - LOGI("[slwr] DealMsg ignore msg, packetId:%u,label=%s,deviceId=%s", - curPacketId, dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - continue; - } - workingId_ = curWorkingId; - LOGI("[slwr] DealMsg workingId_=%u,label=%s,deviceId=%s", workingId_, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - } - int errCode = context_->HandleDataRequestRecv(msg); - delete msg; - msg = nullptr; - { - std::lock_guard lock(lock_); - workingId_ = 0; - bool isWaterMarkErr = context_->IsReceiveWaterMarkErr(); - if (isWaterMarkErr || errCode == -E_NEED_ABILITY_SYNC) { - ClearMap(); - hasFinishedMaxId_ = 0; - endId_ = 0; - isWaterMarkErrHappened_ = true; - } else { - hasFinishedMaxId_++; - hasFinishedPacketId_ = curPacketId; - LOGI("[slwr] DealMsg ok hasFinishedMaxId_=%u,label=%s,deviceId=%s", hasFinishedMaxId_, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - } - context_->SetReceiveWaterMarkErr(false); - workingTaskcv_.notify_all(); - } - } -} - -int SlidingWindowReceiver::TimeOut(TimerId timerId) -{ - { - std::lock_guard lock(lock_); - LOGI("[slwr] TimeOut,timerId[%llu], timerId_[%llu]", timerId, timerId_); - if (timerId == timerId_) { - ClearMap(); - } - timerId_ = 0; - } - RuntimeContext::GetInstance()->RemoveTimer(timerId); - return E_OK; -} - -void SlidingWindowReceiver::StartTimer() -{ - std::lock_guard lock(lock_); - TimerId timerId = 0; - RefObject::IncObjRef(context_); - TimerAction timeOutCallback = std::bind(&SlidingWindowReceiver::TimeOut, this, std::placeholders::_1); - int errCode = RuntimeContext::GetInstance()->SetTimer(IDLE_TIME_OUT, timeOutCallback, - [this]() { - int ret = RuntimeContext::GetInstance()->ScheduleTask([this]() { - RefObject::DecObjRef(context_); - }); - if (ret != E_OK) { - LOGE("[slwr] timer finalizer ScheduleTask, errCode=%d", ret); - } - }, timerId); - if (errCode != E_OK) { - RefObject::DecObjRef(context_); - LOGE("[slwr] timer ScheduleTask, errCode=%d", errCode); - return; - } - StopTimer(timerId_); - timerId_ = timerId; - LOGD("[slwr] StartTimer timerId[%llu] success", timerId_); -} - -void SlidingWindowReceiver::StopTimer() -{ - TimerId timerId; - { - std::lock_guard lock(lock_); - timerId = timerId_; - if (timerId_ == 0) { - return; - } - timerId_ = 0; - } - StopTimer(timerId); -} - -void SlidingWindowReceiver::StopTimer(TimerId timerId) -{ - if (timerId == 0) return; - LOGD("[slwr] StopTimer,remove Timer id[%llu]", timerId); - RuntimeContext::GetInstance()->RemoveTimer(timerId); -} - -void SlidingWindowReceiver::ClearMap() -{ - LOGD("[slwr] ClearMap"); - for (auto &iter : messageMap_) { - delete iter.second; - iter.second = nullptr; - } - messageMap_.clear(); -} - -void SlidingWindowReceiver::ResetInfo() -{ - ClearMap(); - hasFinishedMaxId_ = 0; - hasFinishedPacketId_ = 0; - workingId_ = 0; - endId_ = 0; - isWaterMarkErrHappened_ = false; -} - -int SlidingWindowReceiver::ErrHandle(uint32_t sequenceId, uint64_t packetId) -{ - if (sequenceId == workingId_ || (endId_ != 0 && sequenceId > endId_) - || (packetId > 0 && packetId < hasFinishedPacketId_)) { - LOGI("[slwr] PutMsg sequenceId:%u, endId_:%u, workingId_:%u, packetId:%u!", - sequenceId, endId_, workingId_, packetId); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - // if waterMark err when DealMsg(), the waterMark of msg in messageMap_ is also err, so we clear messageMap_ - // but before the sender receive the waterMark err ack, the receiver may receive waterMark err msg, so before - // receiver receive the sequenceId 1 msg, it will drop msg. - // note, there is a low probability risk, if sender receive the waterMark err ack, it correct the warterMark then - // resend data, if the msg of bigger sequenceId arrive earlier than sequenceId 1, it will be drop, the sender - // will sync timeout. - if (isWaterMarkErrHappened_) { - if (sequenceId == 1) { - isWaterMarkErrHappened_ = false; - } else { - LOGI("[slwr] PutMsg With waterMark Error sequenceId:%u, endId_:%u, workingId_:%u, packetId:%u!", - sequenceId, endId_, workingId_, packetId); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; // drop invalid packet - } - } - return E_OK; -} - -void SlidingWindowReceiver::SetEndField(bool isLastSequence, uint32_t sequenceId) -{ - if (isLastSequence) { - endId_ = sequenceId; - } -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp deleted file mode 100755 index 32df1ca7619976d9d156fa71191e36383496193e..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sliding_window_sender.h" -#include "db_common.h" -namespace DistributedDB { -SlidingWindowSender::~SlidingWindowSender() -{ - Clear(); - context_ = nullptr; - dataSync_ = nullptr; -} - -int SlidingWindowSender::ParamCheck(int32_t mode, const SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - if (context == nullptr) { - LOGE("[slws] SendStart invalid context"); - return -E_INVALID_ARGS; - } - if (dataSync == nullptr) { - LOGE("[slws] SendStart invalid context"); - return -E_INVALID_ARGS; - } - return E_OK; -} - -void SlidingWindowSender::Init(int32_t inMode, SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - isErr_ = false; - mode_ = inMode; - context_->SetSessionEndTimeStamp(0); - context_->ReSetSequenceId(); - maxSequenceIdhasSent_ = 0; - windowSize_ = MAX_WINDOW_SIZE; - int mode = SyncOperation::TransferSyncMode(inMode); - if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL) { - sessionId_ = context->GetRequestSessionId(); - } else { - sessionId_ = context->GetResponseSessionId(); - } -} - -int SlidingWindowSender::Initialize(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync) -{ - if (context == nullptr || dataSync == nullptr) { - LOGE("[slws] Initialize invalid args"); - return -E_INVALID_ARGS; - } - context_ = context; - dataSync_ = dataSync; - return E_OK; -} - -int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - int errCode = ParamCheck(mode, context, dataSync); - if (errCode != E_OK) { - return errCode; - } - errCode = dataSync->CheckPermitSendData(mode, context); - if (errCode != E_OK) { - return errCode; - } - std::lock_guard lock(lock_); - if (sessionId_ != 0) { // auto sync timeout resend - return ReSend(); - } - Init(mode, context, dataSync); - LOGI("[slws] SendStart,mode=%d,label=%s,device=%s", mode_, dataSync->GetLabel().c_str(), - STR_MASK(context->GetDeviceId())); - int tmpMode = SyncOperation::TransferSyncMode(mode); - if (tmpMode == SyncModeType::PUSH) { - errCode = dataSync->PushStart(context); - } else if (tmpMode == SyncModeType::PUSH_AND_PULL) { - errCode = dataSync->PushPullStart(context); - } else if (tmpMode == SyncModeType::PULL) { - errCode = dataSync->PullRequestStart(context); - } else { - errCode = dataSync->PullResponseStart(context); - } - if (context->IsSkipTimeoutError(errCode)) { - // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. - // just return to avoid higher pressure for send. - UpdateInfo(); - return errCode; - } - if (errCode != E_OK) { - LOGE("[slws] SendStart errCode=%d", errCode); - return errCode; - } - errCode = UpdateInfo(); - if (tmpMode == SyncModeType::PUSH_AND_PULL && context_->GetTaskErrCode() == -E_EKEYREVOKED) { - LOGE("wait for recv finished for push and pull mode"); - return -E_EKEYREVOKED; - } - if (errCode == -E_FINISHED) { - return E_OK; - } - return InnerSend(); -} - -int SlidingWindowSender::UpdateInfo() -{ - ReSendInfo reSendInfo; - SyncTimeRange reSendDataTimeRange = context_->GetDataTimeRange(); - reSendInfo.start = reSendDataTimeRange.beginTime; - reSendInfo.end = reSendDataTimeRange.endTime; - reSendInfo.deleteBeginTime = reSendDataTimeRange.deleteBeginTime; - reSendInfo.deleteEndTime = reSendDataTimeRange.deleteEndTime; - reSendInfo.packetId = context_->GetPacketId(); - maxSequenceIdhasSent_++; - reSendMap_[maxSequenceIdhasSent_] = reSendInfo; - windowSize_--; - LOGI("[slws] mode=%d,start=%llu,end=%llu,deleteStart=%llu,deleteEnd=%llu,seqId=%d,packetId=%llu,window_size=%d," - "label=%s,device=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, - reSendInfo.deleteEndTime, maxSequenceIdhasSent_, reSendInfo.packetId, windowSize_, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - ContinueToken token; - context_->GetContinueToken(token); - if (token == nullptr) { - isAllDataHasSent_ = true; - LOGI("[slws] UpdateInfo all data has sent"); - return -E_FINISHED; - } - return E_OK; -} - -int SlidingWindowSender::InnerSend() -{ - while (true) { - if (windowSize_ <= 0 || isAllDataHasSent_) { - LOGI("[slws] InnerSend windowSize_=%d,isAllDataHasSent_=%d", windowSize_, isAllDataHasSent_); - return E_OK; - } - if (isErr_ || context_ == nullptr) { - LOGI("[slws] InnerSend isErr"); - return -E_SLIDING_WINDOW_SENDER_ERR; - } - int errCode; - context_->IncSequenceId(); - int mode = SyncOperation::TransferSyncMode(mode_); - if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { - errCode = dataSync_->PushStart(context_); - } else { - errCode = dataSync_->PullResponseStart(context_); - } - if ((mode == SyncModeType::PUSH_AND_PULL) && errCode == -E_EKEYREVOKED) { - LOGE("errCode = %d, wait for recv finished for push and pull mode", errCode); - isAllDataHasSent_ = true; - return -E_EKEYREVOKED; - } - if (context_->IsSkipTimeoutError(errCode)) { - // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. - // just return to avoid higher pressure for send. - UpdateInfo(); - return errCode; - } - if (errCode != E_OK) { - isErr_ = true; - LOGE("[slws] InnerSend errCode=%d", errCode); - return errCode; - } - errCode = UpdateInfo(); - if (errCode == -E_FINISHED) { - return E_OK; - } - } -} - -int SlidingWindowSender::PreHandleAckRecv(const Message *message) -{ - if (message == nullptr) { - LOGE("[slws] AckRecv message nullptr"); - return -E_INVALID_ARGS; - } - if (message->GetMessageType() == TYPE_NOTIFY || message->IsFeedbackError()) { - return E_OK; - } - const DataAckPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value - std::lock_guard lock(lock_); - uint32_t sequenceId = message->GetSequenceId(); - if (reSendMap_.count(sequenceId) != 0) { - uint64_t originalPacketId = reSendMap_[sequenceId].packetId; - if (DataAckPacket::IsPacketIdValid(packetId) && packetId != originalPacketId) { - LOGE("[slws] packetId[%llu] is not match with original[%llu]", packetId, originalPacketId); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - } - return E_OK; -} - -int SlidingWindowSender::AckRecv(const Message *message) -{ - if (message == nullptr) { - LOGE("[slws] AckRecv message nullptr"); - return -E_INVALID_ARGS; - } - const DataAckPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value - uint32_t sessionId = message->GetSessionId(); - uint32_t sequenceId = message->GetSequenceId(); - LOGI("[slws] AckRecv sequecneId=%d,packetId=%llu,label=%s,dev=%s", sequenceId, packetId, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - - std::lock_guard lock(lock_); - if (sessionId != sessionId_) { - LOGI("[slws] AckRecv sessionId is different"); - return E_OK; - } - if (reSendMap_.count(sequenceId) != 0) { - reSendMap_.erase(sequenceId); - windowSize_++; - LOGI("[slws] AckRecv window_size=%d", windowSize_); - } else { - LOGI("[slws] AckRecv sequenceId not in map"); - return E_OK; - } - if (!isAllDataHasSent_) { - return InnerSend(); - } else if (reSendMap_.size() == 0) { - context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - LOGI("[slws] AckRecv all finished,label=%s,dev=%s", dataSync_->GetLabel().c_str(), - STR_MASK(context_->GetDeviceId())); - InnerClear(); - return -E_FINISHED; - } - return E_OK; -} - -int SlidingWindowSender::ReSend() const -{ - if (isErr_) { - LOGI("[slws] ReSend InnerSend isErr"); - return -E_INTERNAL_ERROR; - } - if (reSendMap_.empty()) { - LOGI("[slws] ReSend map empty"); - return -E_INTERNAL_ERROR; - } - uint32_t sequenceId = reSendMap_.begin()->first; - ReSendInfo reSendInfo = reSendMap_.begin()->second; - LOGI("[slws] ReSend mode=%d,start=%llu,end=%llu,delStart=%llu,delEnd=%llu,seqId=%d,packetId=%llu,windowsize=%d," - "label=%s,deviceId=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, - reSendInfo.deleteEndTime, sequenceId, reSendInfo.packetId, windowSize_, dataSync_->GetLabel().c_str(), - STR_MASK(context_->GetDeviceId())); - DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, - reSendInfo.deleteBeginTime, reSendInfo.deleteEndTime, reSendInfo.packetId}; - return dataSync_->ReSend(context_, dataReSendInfo); -} - -void SlidingWindowSender::Clear() -{ - std::lock_guard lock(lock_); - LOGD("[slws] clear sender info"); - InnerClear(); -} - -void SlidingWindowSender::InnerClear() -{ - sessionId_ = 0; - reSendMap_.clear(); - windowSize_ = 0; - maxSequenceIdhasSent_ = 0; - isAllDataHasSent_ = false; - isErr_ = false; -} - -void SlidingWindowSender::SetErr(bool isErr) -{ - std::lock_guard lock(lock_); - isErr_ = isErr; -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h deleted file mode 100755 index 652a20a1812bdd8e3d60e82c2b512d0c6ffad387..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SLIDING_WINDOW_SENDER_H -#define SLIDING_WINDOW_SENDER_H -#include - -#include "message.h" -#include "single_ver_data_sync.h" -#include "single_ver_sync_task_context.h" - -namespace DistributedDB { -struct ReSendInfo { - TimeStamp start = 0; - TimeStamp end = 0; - TimeStamp deleteBeginTime = 0; - TimeStamp deleteEndTime = 0; - // packetId is used for matched ackpacket packetId which saved in ackPacket.reserve - // if equaled, means need to handle the ack, or drop. it is always increased - uint64_t packetId = 0; -}; - -class SlidingWindowSender { -public: - SlidingWindowSender() = default; - ~SlidingWindowSender(); - DISABLE_COPY_ASSIGN_MOVE(SlidingWindowSender); - - int Initialize(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - // start one dataSync, mode can be push pushAndPull or pullResponse - int SendStart(int32_t mode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int AckRecv(const Message *message); - int PreHandleAckRecv(const Message *message); - void Clear(); - void SetErr(bool isErr); -private: - int ParamCheck(int32_t mode, const SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync); - void Init(int32_t inMode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int UpdateInfo(); - int InnerSend(); - void InnerClear(); - int ReSend() const; // when timeout, used for reSend data - - // initial max window size - static const int MAX_WINDOW_SIZE = 3; - std::mutex lock_; - // 0 is default invalid sessionId. - uint32_t sessionId_ = 0; - // the sequenceId as key - std::map reSendMap_; - // remaining sending window - int32_t windowSize_ = 0; - // in this session, the max sequenceId has been sent - uint32_t maxSequenceIdhasSent_ = 0; - // int this session, all data has been sent, but all ack has received or not - bool isAllDataHasSent_ = false; - SingleVerSyncTaskContext *context_ = nullptr; - std::shared_ptr dataSync_ = nullptr; - int mode_ = 0; // sync mode, e.g. push - bool isErr_ = true; -}; -} // namespace DistributedDB -#endif // SLIDING_WINDOW_SENDER_H - diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp index bdafc83ef993c7028b753148f9500de7a0f44f24..70477fee5854b92e52a6a15e48d90b682e88c26c 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp @@ -269,5 +269,27 @@ std::string SyncOperation::GetQueryId() const { return query_.GetIdentify(); } + + +const std::map &SyncOperation::DBStatusTransMap() +{ + static const std::map transMap = { + { static_cast(OP_FINISHED_ALL), OK }, + { static_cast(OP_TIMEOUT), TIME_OUT }, + { static_cast(OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, + { static_cast(OP_COMM_ABNORMAL), COMM_FAILURE }, + { static_cast(OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, + { static_cast(OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, + { static_cast(OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, + { static_cast(OP_BUSY_FAILURE), BUSY }, + { static_cast(OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, + { static_cast(OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, + { static_cast(OP_NOT_SUPPORT), NOT_SUPPORT }, + { static_cast(OP_INTERCEPT_DATA_FAIL), INTERCEPT_DATA_FAIL }, + { static_cast(OP_MAX_LIMITS), OVER_MAX_LIMITS }, + { static_cast(OP_INVALID_ARGS), INVALID_ARGS }, + }; + return transMap; +} DEFINE_OBJECT_TAG_FACILITIES(SyncOperation) } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h index 547d17a99a96a75cb03de341f1b465ad8169903b..430f23f77352447eb66e6975e8bb19e1931a1098 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h @@ -120,6 +120,8 @@ public: static SyncType GetSyncType(int mode); static int TransferSyncMode(int mode); + static const std::map &DBStatusTransMap(); + protected: virtual ~SyncOperation(); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp index 413d37543a0d6bbc094d8c1d9ea774abc34e8e48..d07877666cd05c140402c4fb8e67a715b555378f 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp @@ -93,10 +93,7 @@ int SyncStateMachine::TimeoutCallback(TimerId timerId) } retryTime++; syncContext_->SetRetryTime(retryTime); - // when version is high than 102, the sequenceid will be managed by slide windows sender. - if (syncContext_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - syncContext_->IncSequenceId(); - } + // the sequenceid will be managed by dataSync slide windows. syncContext_->SetRetryStatus(SyncTaskContext::NEED_RETRY); int timeoutTime = syncContext_->GetSyncRetryTimeout(retryTime); syncContext_->ModifyTimer(timeoutTime); @@ -125,12 +122,6 @@ void SyncStateMachine::Abort() int SyncStateMachine::SwitchMachineState(uint8_t event) { - if (syncContext_->GetRemoteSoftwareVersion() == SOFTWARE_VERSION_EARLIEST) { - currentSyncProctolVersion_ = SINGLE_VER_SYNC_PROCTOL_V2; // ver 101 102 use same table - } else if (syncContext_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_EARLIEST) { - currentSyncProctolVersion_ = syncContext_->GetRemoteSoftwareVersion(); - } - const std::vector &tables = GetStateSwitchTables(); auto tableIter = std::find_if(tables.begin(), tables.end(), [this](const StateSwitchTable &table) { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp index 099a76277aa74bbe22a2f4ccc010534f62931592..909d78c58abb1b42b27905e8706f7e8cf1611660 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp @@ -109,8 +109,8 @@ void SyncTaskContext::SetOperationStatus(int status) LOGD("[SyncTaskContext][SetStatus] syncOperation is null"); return; } - int finalStatus = status; + int operationStatus = syncOperation_->GetStatus(deviceId_); if (status == SyncOperation::OP_SEND_FINISHED && operationStatus == SyncOperation::OP_RECV_FINISHED) { if (GetTaskErrCode() == -E_EKEYREVOKED) { diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 7b98fee3f510bb675998ff3cafe223706c0199cf..0a46a901d0de9dd55fbc67ebd04a8f49aefd6ec7 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -25,14 +25,18 @@ config("module_private_config") { "./unittest/common/interfaces", "../include", "../interfaces/include", + "../interfaces/include/relational", "../interfaces/src", + "../interfaces/src/relational", "../storage/include", "../storage/src", "../storage/src/multiver", "../storage/src/operation", "../storage/src/sqlite", + "../storage/src/sqlite/relational", "../storage/src/upgrader", "../common/include", + "../common/include/relational", "../common/src", "../communicator/include", "../communicator/src", @@ -53,6 +57,7 @@ config("module_private_config") { "LOW_LEVEL_MEM_DEV", "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", + "RELATIONAL_STORE", ] } @@ -85,6 +90,7 @@ ohos_source_set("src_file") { "../common/src/query.cpp", "../common/src/query_expression.cpp", "../common/src/ref_object.cpp", + "../common/src/relational/relational_schema_object.cpp", "../common/src/runtime_context.cpp", "../common/src/runtime_context_impl.cpp", "../common/src/schema_object.cpp", @@ -118,6 +124,13 @@ ohos_source_set("src_file") { "../interfaces/src/kv_store_nb_delegate_impl.cpp", "../interfaces/src/kv_store_result_set_impl.cpp", "../interfaces/src/kv_store_snapshot_delegate_impl.cpp", + "../interfaces/src/relational/data_value.cpp", + "../interfaces/src/relational/relational_store_delegate_impl.cpp", + "../interfaces/src/relational/relational_store_manager.cpp", + "../interfaces/src/relational/relational_store_sqlite_ext.cpp", + "../interfaces/src/relational/runtime_config.cpp", + "../storage/src/data_transformer.cpp", + "../storage/src/db_properties.cpp", "../storage/src/default_factory.cpp", "../storage/src/generic_kvdb.cpp", "../storage/src/generic_kvdb_connection.cpp", @@ -148,10 +161,17 @@ ohos_source_set("src_file") { "../storage/src/operation/multi_ver_database_oper.cpp", "../storage/src/operation/single_ver_database_oper.cpp", "../storage/src/package_file.cpp", + "../storage/src/relational_store_connection.cpp", + "../storage/src/relational_store_instance.cpp", + "../storage/src/relational_sync_able_storage.cpp", + "../storage/src/relationaldb_properties.cpp", "../storage/src/result_entries_window.cpp", "../storage/src/single_ver_natural_store_commit_notify_data.cpp", "../storage/src/sqlite/query_object.cpp", "../storage/src/sqlite/query_sync_object.cpp", + "../storage/src/sqlite/relational/sqlite_relational_store.cpp", + "../storage/src/sqlite/relational/sqlite_relational_store_connection.cpp", + "../storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp", "../storage/src/sqlite/sqlite_local_kvdb.cpp", "../storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "../storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -165,6 +185,8 @@ ohos_source_set("src_file") { "../storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "../storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp", + "../storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "../storage/src/sqlite/sqlite_single_ver_result_set.cpp", "../storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", "../storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", @@ -177,6 +199,7 @@ ohos_source_set("src_file") { "../storage/src/storage_engine.cpp", "../storage/src/storage_engine_manager.cpp", "../storage/src/storage_executor.cpp", + "../storage/src/sync_able_engine.cpp", "../storage/src/sync_able_kvdb.cpp", "../storage/src/sync_able_kvdb_connection.cpp", "../storage/src/upgrader/single_ver_database_upgrader.cpp", @@ -193,18 +216,17 @@ ohos_source_set("src_file") { "../syncer/src/multi_ver_sync_task_context.cpp", "../syncer/src/multi_ver_syncer.cpp", "../syncer/src/query_sync_water_mark_helper.cpp", + "../syncer/src/single_ver_data_message_schedule.cpp", "../syncer/src/single_ver_data_packet.cpp", "../syncer/src/single_ver_data_sync.cpp", - "../syncer/src/single_ver_data_sync_with_sliding_window.cpp", "../syncer/src/single_ver_kv_syncer.cpp", + "../syncer/src/single_ver_relational_syncer.cpp", "../syncer/src/single_ver_serialize_manager.cpp", "../syncer/src/single_ver_sync_engine.cpp", "../syncer/src/single_ver_sync_state_machine.cpp", "../syncer/src/single_ver_sync_target.cpp", "../syncer/src/single_ver_sync_task_context.cpp", "../syncer/src/single_ver_syncer.cpp", - "../syncer/src/sliding_window_receiver.cpp", - "../syncer/src/sliding_window_sender.cpp", "../syncer/src/subscribe_manager.cpp", "../syncer/src/sync_engine.cpp", "../syncer/src/sync_operation.cpp", @@ -362,8 +384,10 @@ distributeddb_unittest("DistributedDBSingleVerP2PSyncTest") { [ "unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp" ] } -distributeddb_unittest("DistributedDBSyncModuleTest") { - sources = [ "unittest/common/syncer/distributeddb_sync_module_test.cpp" ] +distributeddb_unittest("DistributedDBSingleVerMsgScheduleTest") { + sources = [ + "unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp", + ] } distributeddb_unittest("DistributedDBInterfacesNBDelegateTest") { @@ -643,6 +667,7 @@ group("unittest") { ":DistributedDBParcelTest", ":DistributedDBSchemaObjectTest", ":DistributedDBSchemalTest", + ":DistributedDBSingleVerMsgScheduleTest", ":DistributedDBSingleVerP2PQuerySyncTest", ":DistributedDBSingleVerP2PSubscribeSyncTest", ":DistributedDBSingleVerP2PSyncCheckTest", @@ -661,7 +686,6 @@ group("unittest") { ":DistributedDBStorageSingleVerUpgradeTest", ":DistributedDBStorageTransactionDataTest", ":DistributedDBStorageTransactionRecordTest", - ":DistributedDBSyncModuleTest", ":DistributedDBSyncerDeviceManagerTest", ":DistributedDBTimeSyncTest", ":RuntimeContextProcessSystemApiAdapterImplTest", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp index c667a4e22e93c914bc1490541704017780cf41ee..33abc92711ea65812f86ee7344012622a3b1da75 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp @@ -155,6 +155,26 @@ HWTEST_F(DistributedDBCommonTest, SameProcessReUnLockFile, TestSize.Level1) EXPECT_EQ(OS::FileUnlock(fd), E_OK); } +/** + * @tc.name: CalFileSizeTest + * @tc.desc: Test the file size for function test and the performance test. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBCommonTest, CalFileSizeTest, TestSize.Level1) +{ + std::string filePath = g_testDir + "/testFileSize"; + std::ofstream ofs(filePath, std::ofstream::out); + ASSERT_TRUE(ofs.good()); + ofs << "test file size"; + ofs.close(); + uint64_t fileSize = 0; + EXPECT_EQ(OS::CalFileSize(filePath, fileSize), E_OK); + EXPECT_GT(fileSize, 0ULL); + EXPECT_EQ(OS::RemoveFile(filePath), E_OK); +} + // Distributed db is not recommended to use multiple processes to access // This testcase only guard for some wrong use on current product #if defined(RUN_MULTI_PROCESS_TEST) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp index 65de8fe63dcbaa8a9096d999bf6162efaa92ec4d..2080a5e4f7f1e50efb03da1a021ecf485e7282e6 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp @@ -1779,4 +1779,199 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, SingleVerGetSecurityOption002, T EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); g_kvNbDelegatePtr = nullptr; EXPECT_TRUE(g_mgr.DeleteKvStore("SingleVerGetSecurityOption002") == OK); +} + +/** + * @tc.name: MaxLogSize001 + * @tc.desc: Test the pragma cmd of the max log size limit. + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogSize001, TestSize.Level2) +{ + /** + * @tc.steps:step1. Create database. + * @tc.expected: step1. Returns a non-null kvstore. + */ + KvStoreNbDelegate::Option option; + g_mgr.GetKvStore("MaxLogSize001", option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + + /** + * @tc.steps:step2. Setting the max log limit for the valid value. + * @tc.expected: step2. Returns OK. + */ + uint64_t logSize = DBConstant::MAX_LOG_SIZE_HIGH; + PragmaData pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); + + logSize = DBConstant::MAX_LOG_SIZE_LOW; + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); + + logSize = 10 * 1024 * 1024; // 10M + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); + + /** + * @tc.steps:step3. Setting the max log limit for the invalid value. + * @tc.expected: step3. Returns INLIVAD_ARGS. + */ + logSize = DBConstant::MAX_LOG_SIZE_HIGH + 1; + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), INVALID_ARGS); + + logSize = DBConstant::MAX_LOG_SIZE_LOW - 1; + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), INVALID_ARGS); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + g_kvNbDelegatePtr = nullptr; + EXPECT_TRUE(g_mgr.DeleteKvStore("MaxLogSize001") == OK); +} + +/** + * @tc.name: ForceCheckpoint002 + * @tc.desc: Test the checkpoint of the database. + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogSize002, TestSize.Level2) +{ + /** + * @tc.steps:step1. Create database. + * @tc.expected: step1. Returns a non-null kvstore. + */ + KvStoreNbDelegate::Option option; + g_mgr.GetKvStore("MaxLogSize002", option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + + /** + * @tc.steps:step2. Put the random entry into the database. + * @tc.expected: step2. Returns OK. + */ + Key key; + Value value; + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 30); // for 30B random key + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 3 * 1024 * 1024); // 3M value + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 40); // for 40B random key + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 20); // for 20B random key + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 1 * 1024 * 1024); // 1M value + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + + /** + * @tc.steps:step3. Get the resultset. + * @tc.expected: step3. Returns OK. + */ + KvStoreResultSet *resultSet = nullptr; + EXPECT_EQ(g_kvNbDelegatePtr->GetEntries(Key{}, resultSet), OK); + EXPECT_EQ(resultSet->GetCount(), 3); // size of all the entries is 3 + EXPECT_EQ(resultSet->MoveToFirst(), true); + + /** + * @tc.steps:step4. Put more data into the database. + * @tc.expected: step4. Returns OK. + */ + uint64_t logSize = 6 * 1024 * 1024; // 6M for initial test. + PragmaData pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 10); // for 10B random key(different size) + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 3 * 1024 * 1024); // 3MB + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 15); // for 15B random key(different size) + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + + /** + * @tc.steps:step4. Put more data into the database while the log size is over the limit. + * @tc.expected: step4. Returns LOG_OVER_LIMITS. + */ + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 25); // for 25B random key(different size) + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), LOG_OVER_LIMITS); + EXPECT_EQ(g_kvNbDelegatePtr->Delete(key), LOG_OVER_LIMITS); + EXPECT_EQ(g_kvNbDelegatePtr->StartTransaction(), LOG_OVER_LIMITS); + EXPECT_EQ(g_kvNbDelegatePtr->PutLocal(key, value), LOG_OVER_LIMITS); + EXPECT_EQ(g_kvNbDelegatePtr->RemoveDeviceData("deviceA"), LOG_OVER_LIMITS); + /** + * @tc.steps:step4. Change the max log size limit, and put the data. + * @tc.expected: step4. Returns OK. + */ + logSize *= 10; // 10 multiple size + pragLimit = static_cast(&logSize); + EXPECT_EQ(g_kvNbDelegatePtr->Pragma(SET_MAX_LOG_SIZE, pragLimit), OK); + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + g_kvNbDelegatePtr->CloseResultSet(resultSet); + + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("MaxLogSize002"), OK); +} + +/** + * @tc.name: MaxLogCheckPoint001 + * @tc.desc: Pragma the checkpoint command. + * @tc.type: FUNC + * @tc.require: + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBInterfacesNBDelegateTest, MaxLogCheckPoint001, TestSize.Level2) +{ + /** + * @tc.steps:step1. Create database. + * @tc.expected: step1. Returns a non-null kvstore. + */ + KvStoreNbDelegate::Option option; + g_mgr.GetKvStore("MaxLogCheckPoint001", option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + + /** + * @tc.steps:step2. Put the random entry into the database. + * @tc.expected: step2. Returns OK. + */ + Key key; + Value value; + DistributedDBToolsUnitTest::GetRandomKeyValue(key, 30); // for 30B random key(different size) + DistributedDBToolsUnitTest::GetRandomKeyValue(value, 1 * 1024 * 1024); // 1M + EXPECT_EQ(g_kvNbDelegatePtr->Put(key, value), OK); + EXPECT_EQ(g_kvNbDelegatePtr->Delete(key), OK); + + /** + * @tc.steps:step3. Get the disk file size, execute the checkpoint and get the disk file size. + * @tc.expected: step3. Returns OK and the file size is less than the size before checkpoint. + */ + uint64_t sizeBeforeChk = 0; + g_mgr.GetKvStoreDiskSize("MaxLogCheckPoint001", sizeBeforeChk); + EXPECT_GT(sizeBeforeChk, 1 * 1024 * 1024ULL); // more than 1M + int param = 0; + PragmaData paraData = static_cast(¶m); + g_kvNbDelegatePtr->Pragma(EXEC_CHECKPOINT, paraData); + uint64_t sizeAfterChk = 0; + g_mgr.GetKvStoreDiskSize("MaxLogCheckPoint001", sizeAfterChk); + EXPECT_LT(sizeAfterChk, 100 * 1024ULL); // less than 100K + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("MaxLogCheckPoint001"), OK); +} +/** + * @tc.name: CreateMemoryDbWithoutPath + * @tc.desc: Create memory database without path. + * @tc.type: FUNC + * @tc.require: AR000CRAKN + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBInterfacesNBDelegateTest, CreateMemoryDbWithoutPath, TestSize.Level1) +{ + /** + * @tc.steps: step1. Create Memory database by GetKvStore without path. + * @tc.expected: step1. Create successfully. + */ + KvStoreDelegateManager mgr(APP_ID, USER_ID); + const KvStoreNbDelegate::Option option = {true, true}; + mgr.GetKvStore("memory_without_path", option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f0e0baa1f8336623c738adbaba00a0044ccdfe9a --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -0,0 +1,415 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include + +#include "data_transformer.h" +#include "db_common.h" +#include "db_constant.h" +#include "db_errno.h" +#include "db_types.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "generic_single_ver_kv_entry.h" +#include "kvdb_properties.h" +#include "log_print.h" +#include "relational_store_delegate.h" +#include "relational_store_instance.h" +#include "relational_store_manager.h" +#include "relational_store_sqlite_ext.h" +#include "relational_sync_able_storage.h" +#include "sqlite_relational_store.h" +#include "sqlite_utils.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { +string g_testDir; +string g_storePath; +string g_storeID = "dftStoreID"; +const string g_tableName { "data" }; +DistributedDB::RelationalStoreManager g_mgr(APP_ID, USER_ID); +RelationalStoreDelegate *g_delegate = nullptr; + +void CreateDBAndTable() +{ + sqlite3 *db = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + sqlite3_close(db); + return; + } + + const string sql = + "PRAGMA journal_mode=WAL;" + "CREATE TABLE " + g_tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; + char *zErrMsg = nullptr; + errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &zErrMsg); + if (errCode != SQLITE_OK) { + LOGE("sql error:%s", zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_close(db); +} + +int AddOrUpdateRecord(int64_t key, int64_t value) +{ + sqlite3 *db = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode == SQLITE_OK) { + const string sql = + "INSERT OR REPLACE INTO " + g_tableName + " VALUES(" + to_string(key) + "," + to_string(value) + ");"; + errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr); + } + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + sqlite3_close(db); + return errCode; +} + +int GetLogData(int key, uint64_t &flag, TimeStamp ×tamp, const DeviceID &device = "") +{ + string tableName = g_tableName; + if (!device.empty()) { + } + const string sql = "SELECT timestamp, flag \ + FROM " + g_tableName + " as a, " + DBConstant::RELATIONAL_PREFIX + g_tableName + "_log as b \ + WHERE a.key=? AND a.rowid=b.data_key;"; + + sqlite3 *db = nullptr; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + goto END; + } + errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::BindInt64ToStatement(statement, 1, key); // 1 means key's index + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::StepWithRetry(statement, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + timestamp = static_cast(sqlite3_column_int64(statement, 0)); + flag = static_cast(sqlite3_column_int64(statement, 1)); + errCode = E_OK; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = -E_NOT_FOUND; + } + +END: + sqlite3_close(db); + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +void InitStoreProp(const std::string &storePath, const std::string &appId, const std::string &userId, + RelationalDBProperties &properties) +{ + properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); + properties.SetStringProp(RelationalDBProperties::APP_ID, appId); + properties.SetStringProp(RelationalDBProperties::USER_ID, userId); + properties.SetStringProp(RelationalDBProperties::STORE_ID, g_storeID); + std::string identifier = userId + "-" + appId + "-" + g_storeID; + std::string hashIdentifier = DBCommon::TransferHashString(identifier); + properties.SetStringProp(RelationalDBProperties::IDENTIFIER_DATA, hashIdentifier); +} + +const RelationalSyncAbleStorage *GetRelationalStore() +{ + RelationalDBProperties properties; + InitStoreProp(g_storePath, APP_ID, USER_ID, properties); + int errCode = E_OK; + auto store = RelationalStoreInstance::GetDataBase(properties, errCode); + if (store == nullptr) { + LOGE("Get db failed:%d", errCode); + return nullptr; + } + return static_cast(store)->GetStorageEngine(); +} + +int GetCount(sqlite3 *db, const string &sql, size_t &count) +{ + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + return errCode; + } + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + count = static_cast(sqlite3_column_int64(stmt, 0)); + errCode = E_OK; + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} +} + +class DistributedDBRelationalGetDataTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBRelationalGetDataTest::SetUpTestCase(void) +{ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + g_storePath = g_testDir + "/getDataTest.db"; + LOGI("The test db is:%s", g_testDir.c_str()); +} + +void DistributedDBRelationalGetDataTest::TearDownTestCase(void) +{} + +void DistributedDBRelationalGetDataTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + CreateDBAndTable(); +} + +void DistributedDBRelationalGetDataTest::TearDown(void) +{ + if (g_delegate != nullptr) { + EXPECT_EQ(g_mgr.CloseStore(g_delegate), DBStatus::OK); + EXPECT_EQ(g_mgr.DeleteStore(g_storePath), DBStatus::OK); + g_delegate = nullptr; + } + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error."); + } + return; +} + +/** + * @tc.name: LogTbl1 + * @tc.desc: When put sync data to relational store, trigger generate log. + * @tc.type: FUNC + * @tc.require: AR000GK58G + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, LogTbl1, TestSize.Level1) +{ + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); + + /** + * @tc.steps: step1. Put data. + * @tc.expected: Succeed, return OK. + */ + int insertKey = 1; + int insertValue = 1; + EXPECT_EQ(AddOrUpdateRecord(insertKey, insertValue), E_OK); + + /** + * @tc.steps: step2. Check log record. + * @tc.expected: Record exists. + */ + uint64_t flag = 0; + TimeStamp timestamp1 = 0; + EXPECT_EQ(GetLogData(insertKey, flag, timestamp1), E_OK); + EXPECT_EQ(flag, DataItem::LOCAL_FLAG); + EXPECT_NE(timestamp1, 0ULL); +} + +/** + * @tc.name: GetSyncData1 + * @tc.desc: GetSyncData interface + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) +{ + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); + + /** + * @tc.steps: step1. Put 500 records. + * @tc.expected: Succeed, return OK. + */ + const int RECORD_COUNT = 500; + for (int i = 0; i < RECORD_COUNT; ++i) { + EXPECT_EQ(AddOrUpdateRecord(i, i), E_OK); + } + + /** + * @tc.steps: step2. Get all data. + * @tc.expected: Succeed and the count is right. + */ + auto store = GetRelationalStore(); + ASSERT_EQ(store, nullptr); + ContinueToken token = nullptr; + QueryObject query(Query::Select(g_tableName)); + std::vector entries; + DataSizeSpecInfo sizeInfo {MTU_SIZE, 50}; + + int errCode = store->GetSyncData(query, SyncTimeRange {}, sizeInfo, token, entries); + int count = entries.size(); + SingleVerKvEntry::Release(entries); + EXPECT_EQ(errCode, -E_UNFINISHED); + while (token != nullptr) { + errCode = store->GetSyncDataNext(entries, token, sizeInfo); + count += entries.size(); + SingleVerKvEntry::Release(entries); + EXPECT_TRUE(errCode == E_OK || errCode == -E_UNFINISHED); + } + EXPECT_EQ(count, RECORD_COUNT); +} + +/** + * @tc.name: GetQuerySyncData1 + * @tc.desc: GetSyncData interface. + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) +{ + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); + + /** + * @tc.steps: step1. Put 100 records. + * @tc.expected: Succeed, return OK. + */ + const int RECORD_COUNT = 100; // 100 records. + for (int i = 0; i < RECORD_COUNT; ++i) { + EXPECT_EQ(AddOrUpdateRecord(i, i), E_OK); + } + + /** + * @tc.steps: step2. Get data limit 80, offset 30. + * @tc.expected: Get 70 records. + */ + auto store = GetRelationalStore(); + ASSERT_EQ(store, nullptr); + ContinueToken token = nullptr; + const unsigned int LIMIT = 80; // limit as 80. + const unsigned int OFFSET = 30; // offset as 30. + const unsigned int EXPECT_COUNT = RECORD_COUNT - OFFSET; // expect 70 records. + QueryObject query(Query::Select(g_tableName).Limit(LIMIT, OFFSET)); + std::vector entries; + + int errCode = store->GetSyncData(query, SyncTimeRange {}, DataSizeSpecInfo {}, token, entries); + EXPECT_EQ(entries.size(), EXPECT_COUNT); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(token, nullptr); + SingleVerKvEntry::Release(entries); +} + +/** + * @tc.name: GetIncorrectTypeData1 + * @tc.desc: GetSyncData and PutSyncDataWithQuery interface. + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Level1) +{ + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); + + /** + * @tc.steps: step1. Create 2 index for table "data". + * @tc.expected: Succeed, return OK. + */ + sqlite3 *db = nullptr; + ASSERT_EQ(sqlite3_open(g_storePath.c_str(), &db), SQLITE_OK); + string sql = "CREATE INDEX index1 ON " + g_tableName + "(value);" + "CREATE UNIQUE INDEX index2 ON " + g_tableName + "(value,key);"; + ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + + /** + * @tc.steps: step2. Create distributed table "dataPlus". + * @tc.expected: Succeed, return OK. + */ + const string tableName = g_tableName + "Plus"; + sql = "CREATE TABLE " + tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; + ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + ASSERT_EQ(g_delegate->CreateDistributedTable(tableName), DBStatus::OK); + + /** + * @tc.steps: step3. Put 5 records with different type into "dataPlus" table. + * @tc.expected: Succeed, return OK. + */ + vector sqls = { + "INSERT INTO " + tableName + " VALUES(NULL, 1);", + "INSERT INTO " + tableName + " VALUES(NULL, 0.01);", + "INSERT INTO " + tableName + " VALUES(NULL, NULL);", + "INSERT INTO " + tableName + " VALUES(NULL, 'This is a text.');", + "INSERT INTO " + tableName + " VALUES(NULL, x'0123456789');", + }; + const size_t RECORD_COUNT = sqls.size(); + for (const auto &sql : sqls) { + ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + } + + /** + * @tc.steps: step4. Get all data from "dataPlus" table. + * @tc.expected: Succeed and the count is right. + */ + auto store = GetRelationalStore(); + ASSERT_EQ(store, nullptr); + ContinueToken token = nullptr; + QueryObject query(Query::Select(tableName)); + std::vector entries; + EXPECT_EQ(store->GetSyncData(query, SyncTimeRange {}, DataSizeSpecInfo {}, token, entries), E_OK); + EXPECT_EQ(entries.size(), RECORD_COUNT); + + /** + * @tc.steps: step5. Put data into "data" table from deviceA. + * @tc.expected: Succeed, return OK. + */ + QueryObject queryPlus(Query::Select(g_tableName)); + const DeviceID deviceID = "deviceA"; + EXPECT_EQ(const_cast(store)->PutSyncDataWithQuery(queryPlus, entries, deviceID), E_OK); + SingleVerKvEntry::Release(entries); + + /** + * @tc.steps: step6. Check data. + * @tc.expected: All data in the two tables are same. + */ + sql = "SELECT count(*) " + "FROM " + tableName + " as a, " + DBConstant::RELATIONAL_PREFIX + g_tableName + "_" + + DBCommon::TransferStringToHex(DBCommon::TransferHashString(deviceID)) + " as b " + "WHERE a.key=b.key AND (a.value=b.value OR (a.value is NULL AND b.value is NULL));"; + size_t count = 0; + EXPECT_EQ(GetCount(db, sql, count), E_OK); + EXPECT_EQ(count, RECORD_COUNT); + + /** + * @tc.steps: step7. Check index. + * @tc.expected: 2 index for deviceA's data table exists. + */ + sql = "SELECT count(*) FROM sqlite_master WHERE type='index' AND tbl_name='" + DBConstant::RELATIONAL_PREFIX + + g_tableName + "_" + DBCommon::TransferStringToHex(DBCommon::TransferHashString(deviceID)) + "'"; + EXPECT_EQ(GetCount(db, sql, count), E_OK); + EXPECT_EQ(count, 2UL); // The index count is 2. + sqlite3_close(db); +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp index 50496bb84b42b6ca7ce06f88473972954df70f75..f8c8df347e07489c5afd03f66cfb1116ea947263 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp @@ -64,7 +64,7 @@ namespace { std::string GetKvStoreDirectory(const std::string &userId, const std::string &appId, const std::string &storeId) { - string identifier = userId + "-" + appId + "-" + storeId; + string identifier = DBCommon::GenerateIdentifierId(storeId, appId, userId); string hashIdentifierName = DBCommon::TransferHashString(identifier); string identifierName = DBCommon::TransferStringToHex(hashIdentifierName); string filePath = g_testDir + "/" + identifierName + "/" + DBConstant::SINGLE_SUB_DIR + "/main/"; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp index 641c9fe7706ea29ca986623082e19a8a8cf7712d..5244086afdfa1fd26397da2f732f7b12c904119a 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp @@ -1127,8 +1127,8 @@ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest001, TestSize * @tc.steps:step1. Create a query object with table name is specified * @tc.expected: ok */ - Query qeury1 = Query::Select("Relatonal_table").EqualTo("field1", "abc"); - QuerySyncObject obj1(qeury1); + Query query1 = Query::Select("Relatonal_table").EqualTo("field1", "abc"); + QuerySyncObject obj1(query1); /** * @tc.steps:step2. Serialize the object @@ -1158,11 +1158,11 @@ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest001, TestSize */ HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest002, TestSize.Level1) { - Query qeury1 = Query::Select("Relatonal_table1").EqualTo("field1", "abc"); - QuerySyncObject obj1(qeury1); + Query query1 = Query::Select("Relatonal_table1").EqualTo("field1", "abc"); + QuerySyncObject obj1(query1); - Query qeury2 = Query::Select("Relatonal_table2").EqualTo("field1", "abc"); - QuerySyncObject obj2(qeury2); + Query query2 = Query::Select("Relatonal_table2").EqualTo("field1", "abc"); + QuerySyncObject obj2(query2); /** * @tc.steps:step1. check object identity diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index c24f02a1e3bd7619a22ff6498271455bb5342c71..40584d5faa9b26802189e97ab3e35713683fdf58 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -45,12 +45,6 @@ namespace { VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; RelationalVirtualDevice *g_deviceB = nullptr; - // the type of g_kvDelegateCallback is function - auto g_kvDelegateCallback = [](DBStatus status, RelationalStoreDelegate *delegatePtr) { - g_kvDelegateStatus = status; - g_kvDelegatePtr = delegatePtr; - }; - int GetDB(sqlite3 *&db) { int flag = SQLITE_OPEN_URI | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; @@ -61,6 +55,7 @@ namespace { } EXPECT_EQ(SQLiteUtils::RegisterCalcHash(db), E_OK); EXPECT_EQ(SQLiteUtils::RegisterGetSysTime(db), E_OK); + EXPECT_EQ(sqlite3_exec(db, "PRAGMA journal_mode=WAL;", nullptr, nullptr, nullptr), SQLITE_OK); return rc; } @@ -96,7 +91,7 @@ namespace { { sqlite3_stmt *stmt = nullptr; EXPECT_EQ(PrepareInsert(db, stmt), SQLITE_OK); - for (int i = 0; i < (int)rowData.size(); ++i) { + for (int i = 0; i < static_cast(rowData.size()); ++i) { const auto &item = rowData[i]; const int index = i + 1; int errCode; @@ -112,6 +107,7 @@ namespace { EXPECT_EQ(errCode, E_OK); } EXPECT_EQ(SimulateCommitData(db, stmt), SQLITE_DONE); + sqlite3_finalize(stmt); } void GenerateValue(RowData &rowData, std::map &dataMap) @@ -148,21 +144,12 @@ namespace { { std::vector devices = {DEVICE_B}; Query query = Query::Select(g_tableName); - std::mutex syncLock; - std::condition_variable syncCondVar; std::map> statusMap; - std::mutex syncMutex; - std::condition_variable syncCv; - SyncStatusCallback callBack = [&statusMap, &syncCv]( + SyncStatusCallback callBack = [&statusMap]( const std::map> &devicesMap) { statusMap = devicesMap; - syncCv.notify_one(); }; - DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, callBack, query, false); - std::unique_lock uniqueLock(syncMutex); - syncCv.wait(uniqueLock, [&statusMap]() { - return !statusMap.empty(); - }); + DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, query, callBack, true); EXPECT_EQ(callStatus, OK); for (const auto &tablesRes : statusMap) { for (const auto &tableStatus : tablesRes.second) { @@ -185,7 +172,7 @@ namespace { EXPECT_EQ(GetDB(db), SQLITE_OK); EXPECT_EQ(CreateTable(db), SQLITE_OK); - EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName, {}), OK); + EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName), OK); sqlite3_close(db); @@ -206,7 +193,7 @@ namespace { g_deviceB->GetAllSyncData(g_tableName, targetData); for (auto &item : targetData) { - for (int j = 0; j < (int)item.rowData.size(); ++j) { + for (int j = 0; j < static_cast(item.rowData.size()); ++j) { LOGD("index %d actual_val[%s] except_val[%s]", j, item.rowData[j].ToString().c_str(), rowData[j].ToString().c_str()); EXPECT_TRUE(item.rowData[j] == rowData[j]); @@ -230,6 +217,9 @@ void DistributedDBRelationalVerP2PSyncTest::SetUpTestCase() */ DistributedDBToolsUnitTest::TestDirInit(g_testDir); g_dbDir = g_testDir + "/test.db"; + sqlite3 *db = nullptr; + ASSERT_EQ(GetDB(db), SQLITE_OK); + sqlite3_close(db); g_communicatorAggregator = new (std::nothrow) VirtualCommunicatorAggregator(); ASSERT_TRUE(g_communicatorAggregator != nullptr); @@ -254,7 +244,7 @@ void DistributedDBRelationalVerP2PSyncTest::SetUp(void) /** * @tc.setup: create virtual device B, and get a KvStoreNbDelegate as deviceA */ - g_mgr.OpenStore(g_dbDir, {true}, g_kvDelegateCallback); + g_kvDelegateStatus = g_mgr.OpenStore(g_dbDir, "Relational_default_id", {}, g_kvDelegatePtr); ASSERT_TRUE(g_kvDelegateStatus == OK); ASSERT_TRUE(g_kvDelegatePtr != nullptr); g_deviceB = new (std::nothrow) RelationalVirtualDevice(DEVICE_B); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c5c0afa8d159d2cc4b5c268b4cfa449bb3ce3995 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "distributeddb_tools_unit_test.h" +#include "single_ver_data_message_schedule.h" +#include "single_ver_data_packet.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { +} + +class DistributedDBSingleVerMsgScheduleTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBSingleVerMsgScheduleTest::SetUpTestCase(void) +{ +} + +void DistributedDBSingleVerMsgScheduleTest::TearDownTestCase(void) +{ +} + +void DistributedDBSingleVerMsgScheduleTest::SetUp(void) +{ +} + +void DistributedDBSingleVerMsgScheduleTest::TearDown(void) +{ +} + +/** + * @tc.name: MsgSchedule001 + * @tc.desc: Test MsgSchedule funtion with normal sequenceId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule001, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg sequence_3, sequence_2, sequence_1 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 3; i >= 1; i--) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + if (i > 1) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + } + } + /** + * @tc.steps: step2. get msg + * @tc.expected: get msg by sequence_1, sequence_2, sequence_3 + */ + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule002 + * @tc.desc: Test MsgSchedule funtion with by low version + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule002, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg session1_sequence1, session2_sequence1 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_RELEASE_2_0); + DataSyncMessageInfo info; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sessionId_ = i; + info.sequenceId_ = 1; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + /** + * @tc.steps: step2. get msg + * @tc.expected: get msg by session2_sequence1 + */ + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), 1u); + msgSchedule.ScheduleInfoHandle(false, false, msg); + delete msg; + msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule003 + * @tc.desc: Test MsgSchedule funtion with cross sessionId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule003, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg session1_seq1, session2_seq1, session1_seq2, session2_seq2, and handle session1_seq1 + * @tc.expected: handle ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + bool isNeedHandle = true; + bool isNeedContinue = true; + info.sessionId_ = 1; + info.sequenceId_ = 1; + info.packetId_ = 1; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + info.sessionId_ = 2; + info.sequenceId_ = 1; + info.packetId_ = 1; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + info.sessionId_ = 1; + info.sequenceId_ = 2; + info.packetId_ = 2; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + info.sessionId_ = 2; + info.sequenceId_ = 2; + info.packetId_ = 2; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + + /** + * @tc.steps: step2. get msg + * @tc.expected: get msg by session2_seq1, session2_seq2 + */ + for (uint32_t i = 1; i <= 2; i++) { + msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + EXPECT_EQ(msg->GetSessionId(), 2u); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule004 + * @tc.desc: Test MsgSchedule funtion with same sessionId with different packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule004, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq2_packet2, seq3_packet3, seq1_packet4, seq2_packet5, seq3_packet6 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 2; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + } + for (uint32_t i = 1; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i + 3; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + /** + * @tc.steps: step2. get msg + * @tc.expected: drop seq2_packet2, seq3_packet3 and get seq1_packet4, seq2_packet5, seq3_packet6 + */ + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i + 3); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule005 + * @tc.desc: Test MsgSchedule funtion with same sessionId with different packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule005, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq1_packet4, seq2_packet5, seq2_packet2, seq3_packet3 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i + 3; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + for (uint32_t i = 2; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + /** + * @tc.steps: step2. get msg + * @tc.expected: drop seq2_packet2, seq3_packet3 and get seq1_packet4, seq2_packet5 + */ + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i + 3); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule006 + * @tc.desc: Test MsgSchedule funtion with same sessionId and same sequenceId and packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule006, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq1_packet1, seq2_packet2 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + for (uint32_t i = 1; i <= 2; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + /** + * @tc.steps: step2. put msg seq1_packet1, seq2_packet2 again and seq3_packet3 + * @tc.expected: get msg ok and get seq1_packet1, seq2_packet2 and seq3_packet3 + */ + for (uint32_t i = 1; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, (i == 3) ? true : false); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule007 + * @tc.desc: Test MsgSchedule funtion with same sessionId and duplicate sequenceId and low packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule007, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq1_packet4, seq2_packet5 + * @tc.expected: put msg ok and get msg seq1_packet4, seq2_packet5 + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i + 3; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + for (uint32_t i = 1; i <= 2; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i + 3); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + /** + * @tc.steps: step2. put msg seq1_packet1, seq2_packet2 + * @tc.expected: get nullptr + */ + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + EXPECT_EQ(isNeedContinue, true); + ASSERT_TRUE(msg == nullptr); + } + RefObject::KillAndDecObjRef(context); + context = nullptr; +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp deleted file mode 100644 index 2fc18aa7e61d52d5a6ac3eeb731a7678ad4bbf8d..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "db_constant.h" -#include "distributeddb_tools_unit_test.h" -#include "mock_communicator.h" -#include "mock_single_ver_data_sync.h" -#include "mock_single_ver_sync_task_context.h" -#include "sliding_window_receiver.h" - -using namespace testing::ext; -using namespace DistributedDB; -using namespace DistributedDBUnitTest; - -class DistributedDBSyncModuleTest : public testing::Test { -public: - static void SetUpTestCase(void); - static void TearDownTestCase(void); - void SetUp(); - void TearDown(); -}; - -void DistributedDBSyncModuleTest::SetUpTestCase(void) -{ -} - -void DistributedDBSyncModuleTest::TearDownTestCase(void) -{ -} - -void DistributedDBSyncModuleTest::SetUp(void) -{ -} - -void DistributedDBSyncModuleTest::TearDown(void) -{ -} - - -namespace { - void InitMessage(DataSyncMessageInfo &info) - { - info.messageId_ = DistributedDB::DATA_SYNC_MESSAGE; - info.messageType_ = DistributedDB::TYPE_REQUEST; - info.sendCode_ = -DistributedDB::E_UNFINISHED; - info.mode_ = DistributedDB::PUSH; - } - - int BuildPushMessage(DataSyncMessageInfo &info, DistributedDB::Message *&message) - { - return DistributedDBToolsUnitTest::BuildMessage(info, message); - } -} - -/** - * @tc.name: SlidingWindow001 - * @tc.desc: Test sliding window receive data when water error happen - * @tc.type: FUNC - * @tc.require: AR000EV1G6 - * @tc.author: zhangqiquan - */ -HWTEST_F(DistributedDBSyncModuleTest, SlidingWindow001, TestSize.Level1) -{ - std::shared_ptr mockSingleVerDataSync = std::make_shared(); - auto *context = new MockSingleVerSyncTaskContext(); - - SlidingWindowReceiver receiver; - receiver.Initialize(context, mockSingleVerDataSync); - uint32_t sequenceId = 1; - uint32_t sessionId = 1; - uint64_t packedId = 1; - DataSyncMessageInfo info{}; - InitMessage(info); - info.sequenceId_ = sequenceId; - info.sessionId_ = sessionId; - info.packetId_ = packedId; - - DistributedDB::Message *message = nullptr; - int status = BuildPushMessage(info, message); - EXPECT_EQ(status, E_OK); - - /** - * @tc.steps: step1. sliding window receive data first and now water error happen - * @tc.expected: step1. receive msg should return -E_NOT_NEED_DELETE_MSG. - */ - EXPECT_CALL(*context, HandleDataRequestRecv(testing::_)).WillRepeatedly(testing::Return(E_OK)); - EXPECT_CALL(*static_cast(mockSingleVerDataSync.get()), - SendDataAck(testing::_, testing::_, testing::_, testing::_)).WillRepeatedly(testing::Return(E_OK)); - context->SetReceiveWaterMarkErr(true); - EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); - - /** - * @tc.steps: step2. sliding window receive data which is has diff sessionId - * @tc.expected: step2. receive msg should return -E_NOT_NEED_DELETE_MSG. - */ - info.sessionId_++; - status = BuildPushMessage(info, message); - EXPECT_EQ(status, E_OK); - EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); - - /** - * @tc.steps: step3. sliding window receive data which is has diff sequenceId - * @tc.expected: step3. receive msg should return -E_NOT_NEED_DELETE_MSG. - */ - info.sequenceId_++; - info.packetId_++; - status = BuildPushMessage(info, message); - EXPECT_EQ(status, E_OK); - EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); - - receiver.Clear(); - // ensure to call DecRefObj in another thread - std::this_thread::sleep_for(std::chrono::seconds(1)); - RefObject::KillAndDecObjRef(context); - context = nullptr; -} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp index 4771e45222f1a23a479cc998c879997abfe2342c..012e141415abda149c6696bac1a02b1a2c592473 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp @@ -25,7 +25,9 @@ GenericVirtualDevice::GenericVirtualDevice(std::string deviceId) metadata_(nullptr), deviceId_(std::move(deviceId)), remoteDeviceId_("real_device"), - context_(nullptr) + context_(nullptr), + onRemoteDataChanged_(nullptr), + subManager_(nullptr) { } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h index 3605d2cfbf9418c606fe1da7a5bd20365e081bae..388a85152e196f0e34959387268dcc8e5861523d 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h @@ -25,7 +25,6 @@ #include "virtual_communicator_aggregator.h" namespace DistributedDB { -class VirtualCommunicatorAggregator; class GenericVirtualDevice { public: diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp deleted file mode 100644 index eaa1f7764c710163d2eeaf4967bdfb1a492d3c95..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "virtual_device.h" - -#include -#include - -#include "device_manager.h" -#include "log_print.h" -#include "multi_ver_sync_state_machine.h" -#include "single_ver_subscribe_manager.h" -#include "single_ver_sync_state_machine.h" -#include "sync_types.h" -#include "virtual_communicator.h" -#include "virtual_communicator_aggregator.h" -#include "virtual_multi_ver_sync_db_interface.h" -#include "virtual_single_ver_sync_db_Interface.h" - -namespace DistributedDB { -VirtualDevice::VirtualDevice(const std::string &deviceId) - : communicateHandle_(nullptr), - communicatorAggregator_(nullptr), - storage_(nullptr), - metadata_(nullptr), - deviceId_(deviceId), - remoteDeviceId_("real_device"), - context_(nullptr) -{ -} - -VirtualDevice::~VirtualDevice() -{ - std::mutex cvMutex; - std::condition_variable cv; - bool finished = false; - Offline(); - - if (communicateHandle_ != nullptr) { - communicateHandle_->RegOnMessageCallback(nullptr, nullptr); - communicatorAggregator_->ReleaseCommunicator(communicateHandle_); - communicateHandle_ = nullptr; - } - communicatorAggregator_ = nullptr; - - if (context_ != nullptr) { - IKvDBSyncInterface *storage = storage_; - context_->OnLastRef([storage, &cv, &cvMutex, &finished]() { - if (storage != nullptr) { - delete storage; - } - { - std::lock_guard lock(cvMutex); - finished = true; - } - cv.notify_one(); - }); - RefObject::KillAndDecObjRef(context_); - std::unique_lock lock(cvMutex); - cv.wait(lock, [&finished] {return finished;}); - } else { - delete storage_; - } - context_ = nullptr; - metadata_ = nullptr; - storage_ = nullptr; - subManager_ = nullptr; -} - -int VirtualDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface) -{ - if ((communicatorAggregator == nullptr) || (syncInterface == nullptr)) { - return -E_INVALID_ARGS; - } - - communicatorAggregator_ = communicatorAggregator; - int errCode = E_OK; - communicateHandle_ = communicatorAggregator_->AllocCommunicator(deviceId_, errCode); - if (communicateHandle_ == nullptr) { - return errCode; - } - - storage_ = syncInterface; - metadata_ = std::make_shared(); - if (metadata_->Initialize(storage_) != E_OK) { - LOGE("metadata_ init failed"); - return -E_NOT_SUPPORT; - } - if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_SVD) { - context_ = new (std::nothrow) SingleVerSyncTaskContext; - subManager_ = std::make_shared(); - static_cast(context_)->SetSubscribeManager(subManager_); - } else { - context_ = new (std::nothrow) MultiVerSyncTaskContext; - } - if (context_ == nullptr) { - return -E_OUT_OF_MEMORY; - } - communicateHandle_->RegOnMessageCallback(std::bind(&VirtualDevice::MessageCallback, this, - std::placeholders::_1, std::placeholders::_2), []() {}); - context_->Initialize(remoteDeviceId_, storage_, metadata_, communicateHandle_); - context_->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); - context_->RegOnSyncTask(std::bind(&VirtualDevice::StartResponseTask, this)); - return E_OK; -} - -void VirtualDevice::SetDeviceId(const std::string &deviceId) -{ - deviceId_ = deviceId; -} - -std::string VirtualDevice::GetDeviceId() const -{ - return deviceId_; -} - -int VirtualDevice::GetData(const Key &key, VirtualDataItem &item) -{ - VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); - return syncAble->GetSyncData(key, item); -} - -int VirtualDevice::GetData(const Key &key, Value &value) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->GetData(key, value); -} - -int VirtualDevice::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) -{ - VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); - LOGI("dev %s put data time %llu", deviceId_.c_str(), time); - return syncAble->PutData(key, value, time, flag); -} - -int VirtualDevice::PutData(const Key &key, const Value &value) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->PutData(key, value); -} - -int VirtualDevice::DeleteData(const Key &key) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->DeleteData(key); -} - -int VirtualDevice::StartTransaction() -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->StartTransaction(); -} - -int VirtualDevice::Commit() -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->Commit(); -} - -int VirtualDevice::MessageCallback(const std::string &deviceId, Message *inMsg) -{ - if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { - if (onRemoteDataChanged_) { - onRemoteDataChanged_(deviceId); - delete inMsg; - inMsg = nullptr; - return E_OK; - } - delete inMsg; - inMsg = nullptr; - return -E_INVALID_ARGS; - } - - LOGD("[VirtualDevice] onMessag, src %s", deviceId.c_str()); - RefObject::IncObjRef(context_); - RefObject::IncObjRef(communicateHandle_); - SyncTaskContext *context = context_; - ICommunicator *communicateHandle = communicateHandle_; - std::thread thread([context, communicateHandle, inMsg]() { - int errCode = context->ReceiveMessageCallback(inMsg); - if (errCode != -E_NOT_NEED_DELETE_MSG) { - delete inMsg; - } - RefObject::DecObjRef(context); - RefObject::DecObjRef(communicateHandle); - }); - thread.detach(); - return E_OK; -} - -void VirtualDevice::OnRemoteDataChanged(const std::function &callback) -{ - onRemoteDataChanged_ = callback; -} - -void VirtualDevice::Online() -{ - static_cast(communicateHandle_)->Enable(); - communicatorAggregator_->OnlineDevice(deviceId_); -} - -void VirtualDevice::Offline() -{ - static_cast(communicateHandle_)->Disable(); - communicatorAggregator_->OfflineDevice(deviceId_); -} - -int VirtualDevice::StartResponseTask() -{ - LOGD("[VirtualDevice] StartResponseTask"); - RefObject::AutoLock lockGuard(context_); - int status = context_->GetTaskExecStatus(); - if ((status == SyncTaskContext::RUNNING) || context_->IsKilled()) { - LOGD("[VirtualDevice] StartResponseTask status:%d", status); - return -E_NOT_SUPPORT; - } - if (context_->IsTargetQueueEmpty()) { - LOGD("[VirtualDevice] StartResponseTask IsTargetQueueEmpty is empty"); - return E_OK; - } - context_->SetTaskExecStatus(ISyncTaskContext::RUNNING); - context_->MoveToNextTarget(); - LOGI("[VirtualDevice] machine StartSync"); - context_->UnlockObj(); - int errCode = context_->StartStateMachine(); - context_->LockObj(); - if (errCode != E_OK) { - LOGE("[VirtualDevice] machine StartSync failed"); - context_->SetOperationStatus(SyncOperation::OP_FAILED); - } - return errCode; -} - -TimeOffset VirtualDevice::GetLocalTimeOffset() const -{ - return metadata_->GetLocalTimeOffset(); -} - -void VirtualDevice::SetSaveDataDelayTime(uint64_t milliDelayTime) -{ - VirtualSingleVerSyncDBInterface *syncInterface = static_cast(storage_); - syncInterface->SetSaveDataDelayTime(milliDelayTime); -} - -int VirtualDevice::Sync(SyncMode mode, bool wait) -{ - auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} - -int VirtualDevice::Subscribe(QuerySyncObject query, bool wait, int id) -{ - auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, SUBSCRIBE_QUERY, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - operation->SetQuery(query); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} - -int VirtualDevice::UnSubscribe(QuerySyncObject query, bool wait, int id) -{ - auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, UNSUBSCRIBE_QUERY, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - operation->SetQuery(query); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h deleted file mode 100644 index f6d93ff9916609c1876b90af6a5d22ab95fe441d..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VIRTUAL_DEVICE_H -#define VIRTUAL_DEVICE_H - -#include "icommunicator.h" - -#include "ikvdb_sync_interface.h" -#include "single_ver_sync_task_context.h" -#include "virtual_single_ver_sync_db_Interface.h" - -namespace DistributedDB { -class VirtualCommunicatorAggregator; - -class VirtualDevice { -public: - explicit VirtualDevice(const std::string &deviceId); - ~VirtualDevice(); - - int Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface); - void SetDeviceId(const std::string &deviceId); - std::string GetDeviceId() const; - int GetData(const Key &key, VirtualDataItem &item); - int GetData(const Key &key, Value &value); - int PutData(const Key &key, const Value &value, const TimeStamp &time, int flag); - int PutData(const Key &key, const Value &value); - int DeleteData(const Key &key); - int StartTransaction(); - int Commit(); - int MessageCallback(const std::string &deviceId, Message *inMsg); - void OnRemoteDataChanged(const std::function &callback); - void Online(); - void Offline(); - int StartResponseTask(); - TimeOffset GetLocalTimeOffset() const; - void SetSaveDataDelayTime(uint64_t milliDelayTime); - int Sync(SyncMode mode, bool wait); - int Subscribe(QuerySyncObject query, bool wait, int id); - int UnSubscribe(QuerySyncObject query, bool wait, int id); - -private: - ICommunicator *communicateHandle_; - VirtualCommunicatorAggregator *communicatorAggregator_; - IKvDBSyncInterface *storage_; - std::shared_ptr metadata_; - std::string deviceId_; - std::string remoteDeviceId_; - SyncTaskContext *context_; - std::function onRemoteDataChanged_; - std::shared_ptr subManager_; -}; -} // namespace DistributedDB - -#endif // VIRTUAL_DEVICE_H diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp index 37cc0e8c64f48210af751cb48e285b7a26b81a7f..b3efdebe0a2374c55aabb1ed95f2b2e7c4d32b3e 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp @@ -22,7 +22,6 @@ #include "generic_single_ver_kv_entry.h" #include "intercepted_data_impl.h" #include "log_print.h" -#include "meta_data.h" #include "query_object.h" #include "securec.h" diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h new file mode 100644 index 0000000000000000000000000000000000000000..c891afa0b8f944d43e64b79428770f53b345a793 --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H +#define DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H +#include +#include + +#include "relational_store_delegate.h" +#include "relational_store_manager.h" +#include "types.h" +#include "distributed_test_sysinfo.h" +#include "distributeddb_data_generator.h" +#include "log_print.h" +#ifdef TESTCASES_USING_GTEST +#define HWTEST_F(test_case_name, test_name, level) TEST_F(test_case_name, test_name) +#endif + +struct RdbParameters { + std::string path; + std::string storeId; + std::string appId; + std::string userId; + RdbParameters(str::string pathStr, std::string storeIdStr, std::string appIdStr, std::string userIdStr) + : pathStr(pathStr), storeId(storeIdStr), appId(appIdStr), userId(userIdStr) + { + } +}; + +const static std::string TAG = "DistributedRdbTools"; + +const static std::string NORMAL_PATH = "/data/test/"; +const static std::string NON_EXISTENT_PATH = "/data/test/nonExistent_rdb/"; +const static std::string UNREADABLE_PATH = "/data/test/unreadable_rdb/"; +const static std::string UNWRITABLE_PATH = "/data/test/unwritable_rdb/"; + +const static std::string NULL_STOREID = ""; +const static std::string ILLEGAL_STOREID = "rdb_$%#@~%"; +const static std::string MODE_STOREID = "rdb_mode"; +const static std::string FULL_STOREID = "rdb_full"; +const static std::string SUPER_STOREID = "rdb_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +const static std::string NON_EXISTENT_TABLE = "non_table"; +const static std::string KEYWORD_START_TABLE = "naturalbase_rdb_a"; + +const static std::string NORMAL_TABLE = "NORMAL_RDB"; +const static std::string CONSTRAINT_TABLE = "CONSTRAINT_RDB"; + +const static RdbParameters g_rdbParameter1(NORMAL_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter2(NORMAL_PATH, DistributedDBDataGenerator::STORE_ID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter3(NORMAL_PATH, ILLEGAL_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter4(NORMAL_PATH, MODE_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter5(NORMAL_PATH, SUPER_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter6(NON_EXISTENT_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter7(UNREADABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter8(UNWRITABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter9(NORMAL_PATH, FULL_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +class DistributedRdbTools final { +public: + DistributedRdbTools() {} + ~DistributedRdbTools() {} + // Relational Database OpenStore And CreateDistributeTable + static DistributedDB::DBStatus GetOpenStoreStatus(const RelatetionalStoreManager *&manager, + RelatetionalStoreDelegate *&delegate, const RdbParameters ¶m); + static DistributedDB::DBStatus GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, + const std::string &tableName); + static bool CloseStore(const DistributedDB::RelatetionalStoreDelegate *&delegate); + + static bool InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m); + static bool InitTableDataAndTrigger(const sqlite3 *&db) ; + static bool AlterTableAttributes(const sqlite3 *&db); + static bool Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name); + static void CloseSqlite3Store(sqlite3 *&db); +} +#endif // DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H \ No newline at end of file diff --git a/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fae08163e170eaa7eca056e45d3285a89e15a562 --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "distributed_rdb_tools.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef USE_SQLITE_SYMBOLS +#include "sqlite3.h" +#else +#include "sqlite3sym.h" +#endif +#include "distributeddb_data_generator.h" +#include "distributed_test_sysinfo.h" +#include "platform_specific.h" +#include "securec.h" + +using namespace std; +using namespace DistributedDB; +using namespace DistributedDBDataGenerator; + +namespace { + const int MAX_DISTRIBUTED_TABLE_COUNT = 32; + std::string gtableNameList[MAX_DISTRIBUTED_TABLE_COUNT + 1]; + const char *SQL_CREATE_NORMAL_TABLE = "CREATE TABLE IF NOT EXISTS NORMAL_RDB(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "name VARCHAR(100) NOT NULL DEFAULT \"rdb\");"; + + const char *SQL_CREATE_CONSTRAINT_TABLE = "CREATE TABLE IF NOT EXISTS CONSTRAINT_RDB(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "f_id INT," \ + "age INT NOT NULL UNIQUE," \ + "name VARCHAR(100) NOT NULL DEFAULT \"rdb\"," \ + "address CHAR(50) COLLATE NOCASE," \ + "identity INT NOT NULL, CHECK(identity > 10000) PRIMARY KEY(id, identity)," \ + "FOREGIN key(f_id) references NORMAL_RDB(id));"; + + const char *SQL_CREATE_TRIGGER = "CREATE TRIGGER IF NOT EXISTS insertTrigger " \ + "AFTER INSERT ON CONSTRAINT_RDB " \ + "FOR EACH ROW " \ + "BEGIN " \ + "update NORMAL_RDB set name = \"name_001\" " \ + "END"; + + const char *SQL_INSERT_NORMAL_TABLE = "INSERT INTO NORMAL_RDB (id,name)" \ + "VALUES (1, \'rdb_001\'), (2, \'rdb_002\'), (3, \'rdb_003\'), (4, \'rdb_004\'), (5, \'rdb_005\');"; + + + const char *SQL_ADD_FIELD_TABLE1 = "ALTER TABLE RDB_1 ADD COLUMN add_id INI"; + + const char *SQL_ADD_INDEX_TABLE2 = "CREATE INDEX name_index RDB_2 (name)"; + + const char *SQL_DROP_TABLE3 = "DROP TABLE RDB_3"; + + const char *SQL_DROP_CREATE_TABLE3 = "CREATE TABLE IF NOT EXISTS RDB_3(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT);"; + + const char *SQL_DROP_TABLE4 = "DROP TABLE RDB_4"; + + const char *SQL_DROP_CREATE_TABLE4 = "CREATE TABLE IF NOT EXISTS RDB_4(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, name CHAR(100));"; + + const char *SQL_JOURNAL_MODE = "PRAGMA journal_mode = DELETE;"; + + const char *SQL_SYNCHRONOUS_MODE = "PRAGMA synchronous = FULL;"; + + const char *SQL_INSERT_RDB5_TABLE = "INSERT INTO RDB_5 (id,name,age)" \ + "VALUES (1, \'rdb_005\', \'name_rdb5\');"; +} + +DBStatus DistributedRdbTools::GetOpenStoreStatus(const RelatetionalStoreManager *&manager, + RelatetionalStoreDelegate *&delegate, const RdbParameters ¶m) +{ + if (manager == nullptr) { + MST_LOG("%s GetRdbStore failed! manager nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + if (delegate != nullptr) { + delegate = nullptr; + } + DBStatus status = manager->OpenStore(param.path, param.storeId, delegate); + if (delegate == nullptr) { + MST_LOG("%s GetRdbStore failed! delegate nullptr.", TAG.c_str()); + } + retrun status; +} + +DBStatus DistributedRdbTools::GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, + const std::string &tableName) +{ + if (delegate == nullptr) { + MST_LOG("%s CreateDistributedTable failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + DBStatus status = delegate->CreateDistributedTable(tableName); + retrun status; +} + +DBStatus DistributedRdbTools::CloseStore(const RelatetionalStoreDelegate *&delegate) +{ + if (delegate == nullptr) { + MST_LOG("%s CloseStore failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + DBStatus status = delegate->CloseStore(); + return status; +} + +bool DistributedRdbTools::InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m) +{ + const std::string dbName = param.path + param.storeId + ".db"; + DBStributedDB::OS::RemoveFile(dbName); + int errCode = sqlite3_open(dbName, &db); + if (errCode1 != SQLITE_OK) { + MST_LOG("sqlite3_open Failed!"); + return false; + } + errCode = sqlite3_exec(db, "PRAGMA journal_mode = WAL;", 0, 0, 0); + if (errCode != SQLITE_OK) { + MST_LOG("PRAGMA journal_mode = WAL Failed!"); + return false; + } + return true; +} + +namespace { +void SqliteExecSql(sqlite3 *db, const char *sql) +{ + char *errMsg = nullptr; + int errCode = sqlite3_exec(db, sql, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec sql Failed(%s)", errMsg.c_str()); + return false; + } + sqlite3_free(errMsg); + errMsg = nullptr; +} +} + +bool DistributedRdbTools::InitTableDataAndTrigger(const sqlite3 *&db) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + SqliteExecSql(db, SQL_CREATE_NORMAL_TABLE); + SqliteExecSql(db, SQL_CREATE_CONSTRAINT_TABLE); + SqliteExecSql(db, SQL_CREATE_TRIGGER); + SqliteExecSql(db, SQL_INSERT_NORMAL_TABLE); + + for (int i = 1; i <= MAX_DISTRIBUTED_TABLE_COUNT + 1; i++) { + std::string str_0 = "RDB_" + i; + std::string str_1 = "CREATE TABLE IF NOT EXISTS " + std::string str_2 = "( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "name VARCHAR(100) NOT NULL, age VARCHAR(100) NOT NULL);"; + std::string sql = str_1 + str_0 + str_2; + + SqliteExecSql(db, sql.c_str()); + gtableNameList[i-1] = str_0; + } + return true; +} + +bool DistributedRdbTools::AlterTableAttributes(const sqlite3 *&db) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + char *errMsg = nullptr; + int errCode = sqlite3_exec(db, SQL_ADD_FIELD_TABLE1, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_FIELD_TABLE1 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_ADD_INDEX_TABLE2, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_INDEX_TABLE2 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_TABLE3, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_TABLE3 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_CREATE_TABLE3, nullptr, nullptr, &errMsg); + if (errCode1 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_INDEX_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_TABLE4, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_TABLE4 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_CREATE_TABLE4, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_CREATE_TABLE4 Failed(%s)", errMsg.c_str()); + return false; + } + return true; +} + + +bool DistributedRdbTools::Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + int errCode = sqlite3_exec(db, sql_name, 0, 0, 0); + if (errCode != SQLITE_OK) { + MST_LOG("%s Failed!", sql_name); + return false; + } + return true; +} + +void DistributedRdbTools::CloseSqlite3Store(sqlite3 *&db) +{ + if (db != nullptr) { + sqlite3_close(db); + db = nullptr; + } +} \ No newline at end of file diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..530baf215e5fa37d669efd448898fe5632b139b4 --- /dev/null +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "relational_store_delegate.h" +#include "relational_store_manager.h" +#include "distributed_rdb_tools.h" + +using namespace std; +using namespace testing; +#if defined TESTCASES_USING_GTEST_EXT +using namespace testing::ext; +#endif +using namespace DistributedDB; +using namespace DistributedDBDataGenerator; + +// set sqlite3 rdb_A and create 33 dataTables + +namespace DistributedRelatinalDBExceptionOperation { +sqlite3 *gdb = nullptr; + +RelatetionalStoreManager *g_manager = nullptr; + +RelatetionalStoreDelegate *g_delegate = nullptr; + +RelatetionalStoreDelegate *g_delegate1 = nullptr; + +class DistributedRDBExceptionTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +private: +}; + +void DistributedRDBExceptionTest::SetUpTestCase(void) +{ + bool init1 = DistributedRdbTools::InitSqlite3Store(gdb, g_rdbParameter1); + ASSERT_EQ(init1, true); + bool init2 = DistributedRdbTools::InitTableDataAndTrigger(gdb); + ASSERT_EQ(init2, true); + g_manager = new (std::nothrow) RelatetionalStoreManager(g_rdbParameter1.appId, g_rdbParameter1.userId); + DBStatus status1 = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate, g_rdbParameter1); + ASSERT_EQ(status1, DBStatus::OK); +} + +void DistributedRDBExceptionTest::TearDownTestCase(void) +{ + CloseSqlite3Store(gdb); +} + +void DistributedRDBExceptionTest::SetUp(void) +{ + UnitTest *test = UnitTest::GetInstance(); + ASSERT_NE(test, nullptr); + const TestInfo *testinfo = test->current_test_info(); + ASSERT_NE(testinfo, nullptr); + string testCaseName = string(testinfo->name()); + MST_LOG("[SetUp] test case %s is start to run", testCaseName.c_str()); +} + +void DistributedRDBExceptionTest::TearDown(void) +{ + MST_LOG("TearDownTestCase after case."); +} + +/** + * @tc.name: dbPathException001 + * @tc.desc: db path does not exist, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, dbPathException001, TestSize.Level4) +{ + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter6); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: dbPathException002 + * @tc.desc: db path unreadable, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, dbPathException002, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter7); + ASSERT_EQ(init1, true); + bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IXUSR); + ASSERT_EQ(bpermission, true); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter7); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); + bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + CloseSqlite3Store(db); +} + +/** + * @tc.name: dbPathException003 + * @tc.desc: db path unwritdable, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, dbPathException003, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter8); + ASSERT_EQ(init1, true); + bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter8); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); + bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IWUSR | S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + CloseSqlite3Store(db); +} + +/** + * @tc.name: storeIdException001 + * @tc.desc: storeId is empty, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, storeIdException001, TestSize.Level4) +{ + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter2); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: storeIdException002 + * @tc.desc: storeId value larger 128, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, storeIdException002, TestSize.Level4) +{ + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter5); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: storeIdException003 + * @tc.desc: storeId is illegal, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, storeIdException003, TestSize.Level4) +{ + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter3); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: dbModeException001 + * @tc.desc: db mode isn't wal or SYNCHRONOUS full, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, dbModeException001, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter4); + ASSERT_EQ(init1, true); + bool res1 = DistributedRdbTools::Sqlite3ExecOpration(db, SQL_JOURNAL_MODE); + ASSERT_EQ(res1, true); + DBStatus status1 = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + ASSERT_EQ(status1, DBStatus::NOT_SUPPORT); + CloseSqlite3Store(db); + + bool init2 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter9); + ASSERT_EQ(init2, true); + bool res2 = DistributedRdbTools::Sqlite3ExecOpration(db, SQL_SYNCHRONOUS_MODE); + ASSERT_EQ(res2, true); + DBStatus status2 = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); + CloseSqlite3Store(db); +} + +/** + * @tc.name: tableException001 + * @tc.desc: db hasn't non_table, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, tableException001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, NON_EXISTENT_TABLE); + ASSERT_EQ(status2, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: tableException002 + * @tc.desc: db create natrualbase_rdb_A relatinalTable, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, tableException002, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, KEYWORD_START_TABLE); + ASSERT_EQ(status2, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: tableException003 + * @tc.desc: db NORMAL_RDB table has data, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, tableException003, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, NORMAL_TABLE); + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); +} + +/** + * @tc.name: tableMulCreate001 + * @tc.desc: db create RDB_1 RelatinalTable, multiple createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, tableMulCreate001, TestSize.Level4) +{ + for (int i = 1; i <= 5; i++) { + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[0]); + ASSERT_EQ(status2, DBStatus::OK); + } +} + +/** + * @tc.name: mulTableCreate001 + * @tc.desc: dbA createDistributedTable 32 tables, successful + * then dbA createDistributedTable the 33'th table failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, mulTableCreate001, TestSize.Level4) +{ + for (auto tableName: gtableNameList) { + DBStatus status = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, tableName); + if (tableName == gtableNameList[32]) { + ASSERT_EQ(status, DBStatus::NOT_SUPPORT); + } else { + ASSERT_EQ(status, DBStatus::OK); + } + } +} + +/** + * @tc.name: mulTableCreate002 + * @tc.desc: dbA createDistributedTable RDB_1...20 tables, And + * dbB createDistributedTable RDB_21...32 tables,successful + * then dbA createDistributedTable the 33'th table failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, mulTableCreate002, TestSize.Level4) +{ + RelatetionalStoreDelegate *delegateA = nullptr; + RelatetionalStoreDelegate *delegateB = nullptr; + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, delegateA, g_rdbParameter1); + ASSERT_EQ(status, DBStatus::OK); + status = DistributedRdbTools::GetOpenStoreStatus(g_manager, delegateB, g_rdbParameter1); + ASSERT_EQ(status, DBStatus::OK); + for (int index = 0; index < gtableNameList.size(); index++) { + if (index < 20) { + status = DistributedRdbTools::GetCreateDistributedTableStatus(delegateA, gtableNameList[index]); + ASSERT_EQ(status, DBStatus::OK); + } else if (index >= 20 && index < 32) { + status = DistributedRdbTools::GetCreateDistributedTableStatus(delegateB, gtableNameList[index]); + ASSERT_EQ(status, DBStatus::OK); + } else { + ASSERT_EQ(status, DBStatus::NOT_SUPPORT); + } + } +} + +/** + * @tc.name: relatinalTable001 + * @tc.desc: db has RDB_1 relatinalTable, then alter RDB_1 and createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, relatinalTable001, TestSize.Level4) +{ + bool result = DistributedRdbTools::AlterTableAttributes(gdb); + ASSERT_EQ(result, true); + for (int i = 0; i < 4; i++) { + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[i]); + ASSERT_EQ(status2, DBStatus::OK); + } +} + +/** + * @tc.name: constraintTable001 + * @tc.desc: db has constraint table, then createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, constraintTable001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, CONSTRAINT_TABLE); + ASSERT_EQ(status2, DBStatus::OK); +} + +/** + * @tc.name: constraintTable001 + * @tc.desc: db has relatinalTable RDB_5, then add data in local RDB_5 and createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, constraintTable001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + ASSERT_EQ(status2, DBStatus::OK); + bool res1 = DistributedRdbTools::Sqlite3ExecOpration(db, SQL_INSERT_RDB5_TABLE); + ASSERT_EQ(res1, true); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + ASSERT_EQ(status2, DBStatus::OK); +} +} diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp index 538fe699f3d96481d174ad0c0452031b641f23e4..1e196a6b8998e1654b972b844e83d8fba2f383a4 100755 --- a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp @@ -1155,6 +1155,52 @@ HWTEST_F(DistributeddbNbCreateTest, MemoryDb002, TestSize.Level0) ReleaseManager(manager); } +/* + * @tc.name: MemoryDb 003 + * @tc.desc: verify that the memory db do not need path and can create db successfully. + * @tc.type: FUNC + * @tc.require: SR000CRAD8 + * @tc.author: wangyulong + */ +HWTEST_F(DistributeddbNbCreateTest, MemoryDb003, TestSize.Level0) +{ + /** + * @tc.steps: step1. create memory db without path. + * @tc.expected: step1. create successfully. + */ + DelegateMgrNbCallback delegateMgrCallback; + function function = bind(&DelegateMgrNbCallback::Callback, + &delegateMgrCallback, _1, _2); + KvStoreDelegateManager *manager = new (std::nothrow) KvStoreDelegateManager( + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + ASSERT_TRUE(manager != nullptr); + EXPECT_TRUE(manager->SetKvStoreConfig({.dataDir = ""})); + Option optionParam; + optionParam.isMemoryDb = true; + KvStoreNbDelegate::Option option = DistributedDBNbTestTools::TransferNbOptionType(optionParam); + manager->GetKvStore(DistributedDBDataGenerator::STORE_ID_1, option, function); + KvStoreNbDelegate* delegate = const_cast(delegateMgrCallback.GetKvStore()); + ASSERT_TRUE(delegate != nullptr); + + /** + * @tc.steps: step2. put (k1, v1) to the memory db. + * @tc.expected: step2. the memory db can be insert data successfully. + */ + EXPECT_EQ(delegate->PutLocal(KEY_1, VALUE_1), OK); + EXPECT_EQ(delegate->Put(KEY_1, VALUE_2), OK); + Value localValue, syncValue; + EXPECT_EQ(delegate->GetLocal(KEY_1, localValue), OK); + EXPECT_EQ(delegate->Get(KEY_1, syncValue), OK); + EXPECT_EQ(localValue, VALUE_1); + EXPECT_EQ(syncValue, VALUE_2); + + EXPECT_EQ(manager->CloseKvStore(delegate), OK); + delegate = nullptr; + EXPECT_EQ(manager->DeleteKvStore(STORE_ID_1), NOT_FOUND); + + ReleaseManager(manager); +} + /* * @tc.name: OptionParam 001 * @tc.desc: verify that will check the option parameter when create encrypted DB.