diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index c763dcedcbfdd239139cd06b64260be4f4029553..bd6a28f26eebccc0c89404a81dfdbde9b5cee24c 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -18,13 +18,16 @@ config("distrdb_config") { "include", "interfaces/include", "interfaces/src", + "interfaces/src/relational", "common/include", + "common/include/relational", "communicator/include", "storage/include", "storage/src", "storage/src/multiver", "storage/src/operation", "storage/src/sqlite", + "storage/src/sqlite/relational", "storage/src/upgrader", "syncer/include", "syncer/src", @@ -41,6 +44,7 @@ config("distrdb_config") { "USING_DB_JSON_EXTRACT_AUTOMATICALLY", "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", + "RELATIONAL_STORE", ] } @@ -48,6 +52,7 @@ config("distrdb_public_config") { visibility = [ "*:*" ] include_dirs = [ "interfaces/include", + "interfaces/include/relational", "include", ] } @@ -82,6 +87,7 @@ ohos_shared_library("distributeddb") { "common/src/query.cpp", "common/src/query_expression.cpp", "common/src/ref_object.cpp", + "common/src/relational/relational_schema_object.cpp", "common/src/runtime_context.cpp", "common/src/runtime_context_impl.cpp", "common/src/schema_object.cpp", @@ -115,6 +121,13 @@ ohos_shared_library("distributeddb") { "interfaces/src/kv_store_nb_delegate_impl.cpp", "interfaces/src/kv_store_result_set_impl.cpp", "interfaces/src/kv_store_snapshot_delegate_impl.cpp", + "interfaces/src/relational/data_value.cpp", + "interfaces/src/relational/relational_store_delegate_impl.cpp", + "interfaces/src/relational/relational_store_manager.cpp", + "interfaces/src/relational/relational_store_sqlite_ext.cpp", + "interfaces/src/relational/runtime_config.cpp", + "storage/src/data_transformer.cpp", + "storage/src/db_properties.cpp", "storage/src/default_factory.cpp", "storage/src/generic_kvdb.cpp", "storage/src/generic_kvdb_connection.cpp", @@ -145,10 +158,17 @@ ohos_shared_library("distributeddb") { "storage/src/operation/multi_ver_database_oper.cpp", "storage/src/operation/single_ver_database_oper.cpp", "storage/src/package_file.cpp", + "storage/src/relational_store_connection.cpp", + "storage/src/relational_store_instance.cpp", + "storage/src/relational_sync_able_storage.cpp", + "storage/src/relationaldb_properties.cpp", "storage/src/result_entries_window.cpp", "storage/src/single_ver_natural_store_commit_notify_data.cpp", "storage/src/sqlite/query_object.cpp", "storage/src/sqlite/query_sync_object.cpp", + "storage/src/sqlite/relational/sqlite_relational_store.cpp", + "storage/src/sqlite/relational/sqlite_relational_store_connection.cpp", + "storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp", "storage/src/sqlite/sqlite_local_kvdb.cpp", "storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -162,6 +182,8 @@ ohos_shared_library("distributeddb") { "storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp", + "storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "storage/src/sqlite/sqlite_single_ver_result_set.cpp", "storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", "storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", @@ -174,6 +196,7 @@ ohos_shared_library("distributeddb") { "storage/src/storage_engine.cpp", "storage/src/storage_engine_manager.cpp", "storage/src/storage_executor.cpp", + "storage/src/sync_able_engine.cpp", "storage/src/sync_able_kvdb.cpp", "storage/src/sync_able_kvdb_connection.cpp", "storage/src/upgrader/single_ver_database_upgrader.cpp", @@ -190,18 +213,17 @@ ohos_shared_library("distributeddb") { "syncer/src/multi_ver_sync_task_context.cpp", "syncer/src/multi_ver_syncer.cpp", "syncer/src/query_sync_water_mark_helper.cpp", + "syncer/src/single_ver_data_message_schedule.cpp", "syncer/src/single_ver_data_packet.cpp", "syncer/src/single_ver_data_sync.cpp", - "syncer/src/single_ver_data_sync_with_sliding_window.cpp", "syncer/src/single_ver_kv_syncer.cpp", + "syncer/src/single_ver_relational_syncer.cpp", "syncer/src/single_ver_serialize_manager.cpp", "syncer/src/single_ver_sync_engine.cpp", "syncer/src/single_ver_sync_state_machine.cpp", "syncer/src/single_ver_sync_target.cpp", "syncer/src/single_ver_sync_task_context.cpp", "syncer/src/single_ver_syncer.cpp", - "syncer/src/sliding_window_receiver.cpp", - "syncer/src/sliding_window_sender.cpp", "syncer/src/subscribe_manager.cpp", "syncer/src/sync_engine.cpp", "syncer/src/sync_operation.cpp", diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index 99a6944b9081d6801617d966b1cb8d3b0c0158ce..fc0a4e6bb728177099984ecf6e5566cec671c5ba 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -92,6 +92,8 @@ public: static const std::string UPDATE_META_FUNC; + static const std::string SYSTEM_TABLE_PREFIX; + static constexpr uint32_t AUTO_SYNC_TIMEOUT = 5000; // 5s static constexpr uint32_t MANUAL_SYNC_TIMEOUT = 5000; // 5s @@ -109,6 +111,11 @@ public: static constexpr size_t MAX_SYNC_BLOCK_SIZE = 31457280; // 30MB static constexpr int DOUBLE_PRECISION = 15; + static constexpr int MAX_DISTRIBUTED_TABLE_COUNT = 32; + + // For relational + static const std::string RELATIONAL_PREFIX; + static const std::string TIMESTAMP_ALIAS; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h index 5dd9a6e1fcb6c35721e30271ef252590329252e4..f85331478a6dff75ddca738c32118946d3b27c27 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h @@ -115,7 +115,7 @@ constexpr int E_SYSTEM_API_ADAPTER_CALL_FAILED = (E_BASE + 91); // Adapter call constexpr int E_NOT_NEED_DELETE_MSG = (E_BASE + 92); // not need delete msg, will be delete by sliding window receiver constexpr int E_SLIDING_WINDOW_SENDER_ERR = (E_BASE + 93); // sliding window sender err constexpr int E_SLIDING_WINDOW_RECEIVER_INVALID_MSG = (E_BASE + 94); // sliding window receiver invalid msg -constexpr int E_IGNOR_DATA = (E_BASE + 95); // ignore the data changed by other devices and ignore the same data. +constexpr int E_IGNORE_DATA = (E_BASE + 95); // ignore the data changed by other devices and ignore the same data. constexpr int E_FORBID_CACHEDB = (E_BASE + 96); // such after rekey can not check passwd due to file control. constexpr int E_INTERCEPT_DATA_FAIL = (E_BASE + 97); // Intercept push data failed. constexpr int E_INVALID_COMPRESS_ALGO = (E_BASE + 98); // The algo is defined, but there's no implement for the algo. diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_types.h b/services/distributeddataservice/libs/distributeddb/common/include/db_types.h index 697600aa8179561019f8988ddcd8b73d1447e96c..94f6cbd1c86d8c8a435f09842a3b9f22fafbcf77 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_types.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_types.h @@ -123,5 +123,16 @@ enum SingleVerConflictResolvePolicy { DEFAULT_LAST_WIN = 0, DENY_OTHER_DEV_AMEND_CUR_DEV_DATA = 1, }; + +struct SyncTimeRange { + TimeStamp beginTime = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp endTime = static_cast(INT64_MAX); + TimeStamp deleteEndTime = static_cast(INT64_MAX); + bool IsValid() const + { + return (beginTime <= endTime && deleteBeginTime <= deleteEndTime); + } +}; } // namespace DistributedDB #endif // DISTRIBUTEDDB_TYPES_H diff --git a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h index c9ecd26e6e7b1e79d01cf08151e9b1a207039285..ddad4bb797cc9e06032753286abf2a70f83c287e 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h @@ -65,6 +65,8 @@ public: int GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; int GetStringArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; + int GetObjectByFieldPath(const FieldPath &inPath, JsonObject &outObj) const; + // An empty fieldPath indicate the root, the outSubPath should be empty before call, we will not empty it at first. // If inPath is of multiple path, then outSubPath is combination of result of each inPath. int GetSubFieldPath(const FieldPath &inPath, std::set &outSubPath) const; diff --git a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h index 0e4c0dbc8df7b8cb418f397b79735159fa6725bd..ce07d65e1221028e0ebf35c1b134c264fab6e9ac 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h @@ -49,6 +49,8 @@ public: SchemaObject &schemaObject, std::string &canonicalDir); static uint8_t GetValidCompressionRate(uint8_t compressionRate); + + static bool CheckRelationalTableName(const std::string &tableName); }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h index 47e4f99aa591388bd1d5c2d227d95c3d97f6a657..ed59af0742942f815f3cf22ba9347803e519ad47 100644 --- a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -138,7 +138,7 @@ public: const std::map &GetTables() const; - const TableInfo &GetTable(const std::string& tableName) const; + TableInfo GetTable(const std::string& tableName) const; private: int CompareAgainstSchemaObject(const std::string &inSchemaString, std::map &cmpRst) const; @@ -153,6 +153,7 @@ private: int ParseCheckTableInfo(const JsonObject &inJsonObject); int ParseCheckTableName(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableDefine(const JsonObject &inJsonObject, TableInfo &resultTable); + int ParseCheckTableFieldInfo(const JsonObject &inJsonObject, const FieldPath &path, FieldInfo &table); int ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable); int ParseCheckTableIndex(const JsonObject &inJsonObject, TableInfo &resultTable); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp index de9192da34e253fc94c6d0364d84460f19db762e..c04b394063b5c0bffd71fc286d4dbb051f6bb093 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp @@ -290,9 +290,7 @@ std::string DBCommon::GenerateIdentifierId(const std::string &storeId, void DBCommon::SetDatabaseIds(KvDBProperties &properties, const std::string &appId, const std::string &userId, const std::string &storeId) { - properties.SetStringProp(KvDBProperties::APP_ID, appId); - properties.SetStringProp(KvDBProperties::USER_ID, userId); - properties.SetStringProp(KvDBProperties::STORE_ID, storeId); + properties.SetIdentifier(userId, appId, storeId); std::string oriStoreDir; std::string identifier = GenerateIdentifierId(storeId, appId, userId); if (properties.GetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, false)) { @@ -301,7 +299,6 @@ void DBCommon::SetDatabaseIds(KvDBProperties &properties, const std::string &app oriStoreDir = identifier; } std::string hashIdentifier = TransferHashString(identifier); - properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); std::string hashDir = TransferHashString(oriStoreDir); std::string hexHashDir = TransferStringToHex(hashDir); properties.SetStringProp(KvDBProperties::IDENTIFIER_DIR, hexHashDir); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp index 410e011373cf4b6ddf2b3b86f7539d2ecc5dc325..eb90c8a4a93eced1848ae0709995c13f90e3546b 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp @@ -63,4 +63,8 @@ const std::string DBConstant::TRIGGER_REFERENCES_NEW = "NEW."; const std::string DBConstant::TRIGGER_REFERENCES_OLD = "OLD."; const std::string DBConstant::UPDATE_META_FUNC = "update_meta_within_trigger"; + +const std::string DBConstant::SYSTEM_TABLE_PREFIX = "naturalbase_rdb_"; +const std::string DBConstant::RELATIONAL_PREFIX = SYSTEM_TABLE_PREFIX + "aux_"; +const std::string DBConstant::TIMESTAMP_ALIAS = RELATIONAL_PREFIX + "timestamp"; } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index 37a008ddbfc929a2018d09c2f3f6e1f5323ddd2e..5a4d0a01a9f44266e91733130ad8ef09c0a39fb2 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -727,6 +727,7 @@ int JsonObject::GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const { if (!isValid_) { @@ -750,6 +771,7 @@ int JsonObject::GetStringArrayByFieldPath(const FieldPath &inPath, std::vector -#include #include "json_object.h" #include "relational_schema_object.h" +#include "schema_utils.h" namespace DistributedDB { const std::string &FieldInfo::GetFieldName() const @@ -97,15 +97,13 @@ void FieldInfo::SetColumnId(int cid) // return field define string like ("fieldName": "MY INT(21), NOT NULL, DEFAULT 123") std::string FieldInfo::ToAttributeString() const { - std::string attrStr = "\"" + fieldName_ + "\": \""; - attrStr += dataType_; - if (isNotNull_) { - attrStr += ", NOT NULL"; - } + std::string attrStr = "\"" + fieldName_ + "\": {"; + attrStr += "\"TYPE\":\"" + dataType_ + "\","; + attrStr += "\"NOT_NULL\":" + std::string(isNotNull_ ? "true" : "false") + ","; if (hasDefaultValue_) { - attrStr += ", " + defaultValue_; + attrStr += "\"DEFAULT\":\"" + defaultValue_ + "\""; } - attrStr += + "\""; + attrStr += "}"; return attrStr; } @@ -270,7 +268,7 @@ void TableInfo::SetDevId(const std::string &devId) } namespace { - std::string VectorJoin(const CompositeFields& fields, char con) + std::string VectorJoin(const CompositeFields &fields, char con) { std::string res; auto it = fields.begin(); @@ -292,13 +290,12 @@ JsonObject TableInfo::ToJsonObject() const tableJson.InsertField(FieldPath { "AUTOINCREMENT" }, FieldType::LEAF_FIELD_BOOL, jsonField); jsonField.stringValue = primaryKey_; tableJson.InsertField(FieldPath { "PRIMARY_KEY" }, FieldType::LEAF_FIELD_STRING, jsonField); - for (const auto& it : fields_) { + for (const auto &it : fields_) { jsonField.stringValue = it.second.ToAttributeString(); tableJson.InsertField(FieldPath { "DEFINE", it.first }, FieldType::LEAF_FIELD_STRING, jsonField); } - for (const auto& it : uniqueDefines_) { + for (const auto &it : uniqueDefines_) { jsonField.stringValue = VectorJoin(it, ','); - // TODO: add unique to tableJson } return tableJson; } @@ -337,7 +334,7 @@ int RelationalSyncOpinion::Deserialization(const Parcel &parcel) return E_OK; } -const SyncOpinion &RelationalSyncOpinion::GetTableOpinion(const std::string& tableName) const +const SyncOpinion &RelationalSyncOpinion::GetTableOpinion(const std::string &tableName) const { return opinions_.at(tableName); } @@ -408,12 +405,12 @@ int RelationalSchemaObject::ParseFromSchemaString(const std::string &inSchemaStr } schemaType_ = SchemaType::RELATIVE; - schemaString_ = inSchemaString; + schemaString_ = schemaObj.ToString(); isValid_ = true; return E_OK; } -void RelationalSchemaObject::AddRelationalTable(const TableInfo& tb) +void RelationalSchemaObject::AddRelationalTable(const TableInfo &tb) { tables_[tb.GetTableName()] = tb; } @@ -423,9 +420,13 @@ const std::map &RelationalSchemaObject::GetTables() cons return tables_; } -const TableInfo &RelationalSchemaObject::GetTable(const std::string& tableName) const +TableInfo RelationalSchemaObject::GetTable(const std::string &tableName) const { - return tables_.at(tableName); + auto it = tables_.find(tableName); + if (it != tables_.end()) { + return it->second; + } + return {}; } int RelationalSchemaObject::CompareAgainstSchemaObject(const std::string &inSchemaString, @@ -444,8 +445,9 @@ namespace { int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &fieldName, FieldType expectType, bool isNecessary, FieldValue &fieldValue) { - if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName}) && !isNecessary) { - return E_OK; + if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName})) { + LOGW("[RelationalSchema][Parse] Get schema %s not exist. isNecessary: %d", fieldName.c_str(), isNecessary); + return isNecessary ? -E_SCHEMA_PARSE_FAIL : -E_NOT_FOUND; } FieldType fieldType; @@ -456,11 +458,11 @@ int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &f } if (fieldType != expectType) { - LOGE("[RelationalSchema][Parse] Expect %s fieldType %d but : %d.", fieldName.c_str(), expectType, fieldType); + LOGE("[RelationalSchema][Parse] Expect %s fieldType %d but: %d.", fieldName.c_str(), expectType, fieldType); return -E_SCHEMA_PARSE_FAIL; } - errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {"NAME"}, fieldValue); + errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {fieldName}, fieldValue); if (errCode != E_OK) { LOGE("[RelationalSchema][Parse] Get schema %s value failed: %d.", fieldName.c_str(), errCode); return -E_SCHEMA_PARSE_FAIL; @@ -479,11 +481,7 @@ int RelationalSchemaObject::ParseRelationalSchema(const JsonObject &inJsonObject if (errCode != E_OK) { return errCode; } - errCode = ParseCheckSchemaTableDefine(inJsonObject); - if (errCode != E_OK) { - return errCode; - } - return E_OK; + return ParseCheckSchemaTableDefine(inJsonObject); } int RelationalSchemaObject::ParseCheckSchemaVersionMode(const JsonObject &inJsonObject) @@ -596,46 +594,67 @@ int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject } for (const auto &field : tableFields) { - if (field.second != FieldType::LEAF_FIELD_STRING) { - LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType STRING but : %s.", + if (field.second != FieldType::INTERNAL_FIELD_OBJECT) { + LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType INTERNAL OBJECT but : %s.", SchemaUtils::FieldTypeString(field.second).c_str()); return -E_SCHEMA_PARSE_FAIL; } - FieldValue fieldValue; - errCode = inJsonObject.GetFieldValueByFieldPath(field.first, fieldValue); - if (errCode != E_OK) { - LOGE("[RelationalSchema][Parse] Get schema TABLES DEFINE field value failed: %d.", errCode); - return -E_SCHEMA_PARSE_FAIL; - } - SchemaAttribute outAttr; - errCode = SchemaUtils::ParseAndCheckSchemaAttribute(fieldValue.stringValue, outAttr, false); + JsonObject fieldObj; + errCode = inJsonObject.GetObjectByFieldPath(field.first, fieldObj); if (errCode != E_OK) { - LOGE("[RelationalSchema][Parse] Parse schema TABLES DEFINE attribute failed: %d.", errCode); + LOGE("[RelationalSchema][Parse] Get table field object failed. %d", errCode); return errCode; } FieldInfo fieldInfo; - fieldInfo.SetFieldName(field.first[1]); - fieldInfo.SetDataType(outAttr.customFieldType); - fieldInfo.SetNotNull(outAttr.hasNotNullConstraint); - if (outAttr.hasDefaultValue) { - fieldInfo.SetDefaultValue(outAttr.defaultValue.stringValue); + fieldInfo.SetFieldName(field.first[0]); // 0 : first element in path + errCode = ParseCheckTableFieldInfo(fieldObj, field.first, fieldInfo); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Parse table field info failed. %d", errCode); + return -E_SCHEMA_PARSE_FAIL; } resultTable.AddField(fieldInfo); } return E_OK; } +int RelationalSchemaObject::ParseCheckTableFieldInfo(const JsonObject &inJsonObject, const FieldPath &path, + FieldInfo &table) +{ + FieldValue fieldValue; + int errCode = GetMemberFromJsonObject(inJsonObject, "TYPE", FieldType::LEAF_FIELD_STRING, true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + table.SetDataType(fieldValue.stringValue); + + errCode = GetMemberFromJsonObject(inJsonObject, "NOT_NULL", FieldType::LEAF_FIELD_BOOL, true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + table.SetNotNull(fieldValue.boolValue); + + errCode = GetMemberFromJsonObject(inJsonObject, "DEFAULT", FieldType::LEAF_FIELD_STRING, false, fieldValue); + if (errCode == E_OK) { + table.SetDefaultValue(fieldValue.stringValue); + } else if (errCode != -E_NOT_FOUND) { + return errCode; + } + + return E_OK; +} + int RelationalSchemaObject::ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable) { FieldValue fieldValue; - int errCode = GetMemberFromJsonObject(inJsonObject, "AUTOINCREMENT", FieldType::LEAF_FIELD_BOOL, - false, fieldValue); + int errCode = GetMemberFromJsonObject(inJsonObject, "AUTOINCREMENT", FieldType::LEAF_FIELD_BOOL, false, fieldValue); if (errCode == E_OK) { resultTable.SetAutoIncrement(fieldValue.boolValue); + } else if (errCode != -E_NOT_FOUND) { + return errCode; } - return errCode; + return E_OK; } int RelationalSchemaObject::ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable) @@ -681,7 +700,7 @@ int RelationalSchemaObject::ParseCheckTableIndex(const JsonObject &inJsonObject, LOGE("[RelationalSchema][Parse] Get schema TABLES INDEX field value failed: %d.", errCode); return -E_SCHEMA_PARSE_FAIL; } - resultTable.AddIndexDefine(field.first[1], indexDefine); + resultTable.AddIndexDefine(field.first[1], indexDefine); // 1 : second element in path } return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h index ea7376a1a1b723b7e90e26e992947027b3972f92..4fba674560c566bb89bd14f08f733953daf16149 100755 --- a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h +++ b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h @@ -46,6 +46,7 @@ struct AutoLaunchParam { std::string storeId; AutoLaunchOption option; AutoLaunchNotifier notifier; + std::string path; }; using AutoLaunchRequestCallback = std::function; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h index bd50513ae27efc01e8275eeb3cc8747b1277f078..50bfc8485b870cec0be261c3e3b90bc5e3c68081 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h @@ -37,7 +37,10 @@ public: Blob(); ~Blob(); - DISABLE_COPY_ASSIGN_MOVE(Blob); + Blob(Blob &&); + Blob(const Blob &) = delete; + Blob &operator=(Blob &&); + Blob &operator=(const Blob &) = delete; const uint8_t* GetData() const; uint32_t GetSize() const; @@ -65,6 +68,8 @@ public: DataValue &operator=(const double &doubleVal); DataValue &operator=(const Blob &blob); DataValue &operator=(const std::string &string); + int Set(Blob *&blob); + // equals bool operator==(const DataValue &dataValue) const; bool operator!=(const DataValue &dataValue) const; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h index 3e2923b77680b6f0b0bbd3b4eb638fb4b33078f9..fbf038564aa8476dc32b4d2dd0015ae3b7c0244e 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -27,23 +27,20 @@ public: DB_API virtual ~RelationalStoreDelegate() = default; struct Option { - bool createIfNecessary = true; + // split mode }; DB_API virtual DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) = 0; DB_API virtual DBStatus RemoveDeviceData(const std::string &device) = 0; - struct TableOption { - }; - - DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) = 0; + DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName) = 0; DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) = 0; DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait) = 0; + const Query &query, SyncStatusCallback &onComplete, bool wait) = 0; DB_API virtual DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) = 0; }; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h index 17f669b0033f50eb6e232edfffdfd8709601a8c0..1fed6cbd21b345de475d8c9a9144c709a1386593 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -21,8 +21,6 @@ #include "auto_launch_export.h" #include "relational_store_delegate.h" -#include "irelational_store.h" -#include "kvdb_properties.h" #include "types.h" namespace DistributedDB { @@ -36,16 +34,18 @@ public: RelationalStoreManager &operator=(const RelationalStoreManager &) = delete; RelationalStoreManager &operator=(RelationalStoreManager &&) = delete; - // PRAGMA journal_mode=WAL - // PRAGMA synchronous=FULL - // PRAGMA synchronous=NORMAL - DB_API void OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, - const std::function &callback); + DB_API DBStatus OpenStore(const std::string &path, const std::string &storeId, + const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate); DB_API DBStatus CloseStore(RelationalStoreDelegate *store); DB_API DBStatus DeleteStore(const std::string &path); + DB_API static void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback); + + DB_API static std::string GetRelationalStoreIdentifier(const std::string &userId, const std::string &appId, + const std::string &storeId); + private: std::string appId_; std::string userId_; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h index fb12ab170a7a1f206c361a3c0d3c0119485dbc13..dca53003d252938f19ef9ef874f251790714d7a5 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h @@ -56,6 +56,8 @@ enum DBStatus { SECURITY_OPTION_CHECK_ERROR, // such as remote device's SecurityOption not equal to local SCHEMA_VIOLATE_VALUE, // Values already exist in dbFile do not match new schema INTERCEPT_DATA_FAIL, // Interceptor push data failed. + RELATIONAL_SCHEMA_NOT_FOUND, // the sync table is not a relational table + RELATIONAL_SCHEMA_CHANGED, // the schema was changed }; struct KvStoreConfig { diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp index ad5072181e94e7df2f36a12ac6d779775fc5fc78..f7405885cb9d5b7630395b389ad1962f69edfb7c 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -33,6 +33,21 @@ Blob::~Blob() size_ = 0; } +Blob::Blob(Blob &&blob) : ptr_(blob.ptr_), size_(blob.size_) +{ + blob.ptr_ = nullptr; + blob.size_ = 0; +} + +Blob &Blob::operator=(Blob &&blob) +{ + ptr_ = blob.ptr_; + size_ = blob.size_; + blob.ptr_ = nullptr; + blob.size_ = 0; + return *this; +} + const uint8_t *Blob::GetData() const { return ptr_; @@ -46,7 +61,7 @@ uint32_t Blob::GetSize() const int Blob::WriteBlob(const uint8_t *ptrArray, const uint32_t &size) { if (size == 0) return E_OK; - ptr_ = new(std::nothrow) uint8_t[size]; + ptr_ = new (std::nothrow) uint8_t[size]; if (ptr_ == nullptr) { return -E_OUT_OF_MEMORY; } @@ -159,7 +174,7 @@ DataValue &DataValue::operator=(const Blob &blob) if (blob.GetSize() <= 0) { return *this; } - value_.blobPtr = new(std::nothrow) Blob(); + value_.blobPtr = new (std::nothrow) Blob(); if (value_.blobPtr == nullptr) { return *this; } @@ -170,10 +185,23 @@ DataValue &DataValue::operator=(const Blob &blob) return *this; } +int DataValue::Set(Blob *&blob) +{ + ResetValue(); + if (blob->GetSize() <= 0) { + LOGE("Transfer Blob to DataValue failed."); + return -E_INVALID_ARGS; + } + type_ = StorageType::STORAGE_TYPE_BLOB; + value_.blobPtr = blob; + blob = nullptr; + return E_OK; +} + DataValue &DataValue::operator=(const std::string &string) { ResetValue(); - value_.blobPtr = new(std::nothrow) Blob(); + value_.blobPtr = new (std::nothrow) Blob(); if (value_.blobPtr == nullptr) { return *this; } @@ -247,7 +275,7 @@ int DataValue::GetBlob(Blob *&outVal) const if (type_ != StorageType::STORAGE_TYPE_BLOB && type_ != StorageType::STORAGE_TYPE_TEXT) { return -E_NOT_SUPPORT; } - outVal = new(std::nothrow) Blob(); + outVal = new (std::nothrow) Blob(); if (outVal == nullptr) { return -E_OUT_OF_MEMORY; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp index fb4d08542149f6b884cfd04c2c6f95aa89023c56..fd81ccee0cfbf3a061061f3ebe15683672604ef9 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -16,8 +16,9 @@ #include "relational_store_delegate_impl.h" #include "db_errno.h" -#include "log_print.h" #include "kv_store_errno.h" +#include "log_print.h" +#include "param_check_utils.h" #include "sync_operation.h" namespace DistributedDB { @@ -52,15 +53,19 @@ DBStatus RelationalStoreDelegateImpl::RemoveDeviceData(const std::string &device return NOT_SUPPORT; } -DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName, const TableOption &option) +DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName) { - // check table Name and option + if (!ParamCheckUtils::CheckRelationalTableName(tableName)) { + LOGE("invalid table name."); + return INVALID_ARGS; + } + if (conn_ == nullptr) { LOGE("[RelationalStore Delegate] Invalid connection for operation!"); return DB_ERROR; } - int errCode = conn_->CreateDistributedTable(tableName, option); + int errCode = conn_->CreateDistributedTable(tableName); if (errCode != E_OK) { LOGW("[RelationalStore Delegate] Create Distributed table failed:%d", errCode); return TransferDBErrno(errCode); @@ -69,14 +74,15 @@ DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string & } DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait) + const Query &query, SyncStatusCallback &onComplete, bool wait) { if (conn_ == nullptr) { LOGE("Invalid connection for operation!"); return DB_ERROR; } - RelationalStoreConnection::SyncInfo syncInfo{devices, mode, onComplete, query, wait}; + RelationalStoreConnection::SyncInfo syncInfo{devices, mode, + std::bind(&RelationalStoreDelegateImpl::OnSyncComplete, std::placeholders::_1, onComplete), query, wait}; int errCode = conn_->SyncToDevice(syncInfo); if (errCode != E_OK) { LOGW("[RelationalStore Delegate] sync data to device failed:%d", errCode); @@ -115,5 +121,43 @@ void RelationalStoreDelegateImpl::SetReleaseFlag(bool flag) { releaseFlag_ = flag; } + +void RelationalStoreDelegateImpl::OnSyncComplete(const std::map> &devicesMap, + SyncStatusCallback &onComplete) +{ + static const std::map statusMap = { + { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, + { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, + { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, + { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, + { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, + { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, + { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, + { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, + { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, + { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, + { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, + { static_cast(SyncOperation::OP_INTERCEPT_DATA_FAIL), INTERCEPT_DATA_FAIL }, + { static_cast(SyncOperation::OP_MAX_LIMITS), OVER_MAX_LIMITS }, + { static_cast(SyncOperation::OP_INVALID_ARGS), INVALID_ARGS }, + }; + std::map> res; + for (const auto &[device, statusList] : devicesMap) { + for (const auto &tableStatus : statusList) { + TableStatus table; + table.tableName = tableStatus.tableName; + DBStatus status = DB_ERROR; + auto iterator = statusMap.find(tableStatus.status); + if (iterator != statusMap.end()) { + status = iterator->second; + } + table.status = status; + res[device].push_back(table); + } + } + if (onComplete) { + onComplete(res); + } +} } // namespace DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h index 6dc5404c254ab98da64c2fe61ff2f078fcb349f6..19911f74bd23a8fc77894180327ee47279a9905d 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -33,12 +33,14 @@ public: DBStatus Sync(const std::vector &devices, SyncMode mode, SyncStatusCallback &onComplete, bool wait) override; + DBStatus Sync(const std::vector &devices, SyncMode mode, - SyncStatusCallback &onComplete, const Query &query, bool wait) override; + const Query &query, SyncStatusCallback &onComplete, bool wait) override; DBStatus RemoveDeviceData(const std::string &device) override; - DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) override; + DBStatus CreateDistributedTable(const std::string &tableName) override; + DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) override; // For connection @@ -46,6 +48,9 @@ public: void SetReleaseFlag(bool flag); private: + static void OnSyncComplete(const std::map> &devicesMap, + SyncStatusCallback &onComplete); + RelationalStoreConnection *conn_ = nullptr; std::string storePath_; std::atomic releaseFlag_ = false; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h index 34316ef8260ab7d4c26fe4d75892f1d7c08742d1..5c24c1f2cf1b0824b39352380a113e18b2721bfa 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h @@ -20,7 +20,7 @@ #include #include "irelational_store.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" namespace DistributedDB { class RelationalStoreInstance final { @@ -28,19 +28,19 @@ public: RelationalStoreInstance(); ~RelationalStoreInstance() = default; - static RelationalStoreConnection *GetDatabaseConnection(const DBProperties &properties, int &errCode); + static RelationalStoreConnection *GetDatabaseConnection(const RelationalDBProperties &properties, int &errCode); static RelationalStoreInstance *GetInstance(); int CheckDatabaseFileStatus(const std::string &id); // public for test mock - static IRelationalStore *GetDataBase(const DBProperties &properties, int &errCode); + static IRelationalStore *GetDataBase(const RelationalDBProperties &properties, int &errCode); private: - IRelationalStore *OpenDatabase(const DBProperties &properties, int &errCode); + IRelationalStore *OpenDatabase(const RelationalDBProperties &properties, int &errCode); - void RemoveKvDBFromCache(const DBProperties &properties); - void SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties); + void RemoveKvDBFromCache(const RelationalDBProperties &properties); + void SaveKvDBToCache(IRelationalStore *store, const RelationalDBProperties &properties); static RelationalStoreInstance *instance_; static std::mutex instanceLock_; diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp index 9a4a60324bc34d0cbdf3c274cbb73157e836b645..d2a3bd5ccd5af671220da61e380c8b4d145bbc9b 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -17,6 +17,7 @@ #include +#include "auto_launch.h" #include "relational_store_instance.h" #include "db_common.h" #include "param_check_utils.h" @@ -24,6 +25,7 @@ #include "db_errno.h" #include "kv_store_errno.h" #include "relational_store_delegate_impl.h" +#include "runtime_context.h" #include "platform_specific.h" namespace DistributedDB { @@ -32,23 +34,17 @@ RelationalStoreManager::RelationalStoreManager(const std::string &appId, const s userId_(userId) {} -static void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, - const std::string &appId, const std::string &userId, DBProperties &properties) +void InitStoreProp(const std::string &storePath, const std::string &appId, const std::string &userId, + const std::string &storeId, RelationalDBProperties &properties) { - properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); - properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); - properties.SetStringProp(KvDBProperties::APP_ID, appId); - properties.SetStringProp(KvDBProperties::USER_ID, userId); - properties.SetStringProp(KvDBProperties::STORE_ID, storePath); // same as dir - std::string identifier = userId + "-" + appId + "-" + storePath; - std::string hashIdentifier = DBCommon::TransferHashString(identifier); - properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); + properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); + properties.SetIdentifier(userId, appId, storeId); } static const int GET_CONNECT_RETRY = 3; static const int RETRY_GET_CONN_INTER = 30; -static RelationalStoreConnection *GetOneConnectionWithRetry(const DBProperties &properties, int &errCode) +static RelationalStoreConnection *GetOneConnectionWithRetry(const RelationalDBProperties &properties, int &errCode) { for (int i = 0; i < GET_CONNECT_RETRY; i++) { auto conn = RelationalStoreInstance::GetDatabaseConnection(properties, errCode); @@ -64,39 +60,38 @@ static RelationalStoreConnection *GetOneConnectionWithRetry(const DBProperties & return nullptr; } -void RelationalStoreManager::OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, - const std::function &callback) +DB_API DBStatus RelationalStoreManager::OpenStore(const std::string &path, const std::string &storeId, + const RelationalStoreDelegate::Option &option, RelationalStoreDelegate *&delegate) { - if (!callback) { - LOGE("[KvStoreMgr] Invalid callback for kv store!"); - return; + if (delegate != nullptr) { + LOGE("[RelationalStoreMgr] Invalid delegate!"); + return INVALID_ARGS; + } + + std::string canonicalDir; + if (!ParamCheckUtils::CheckDataDir(path, canonicalDir)) { + return INVALID_ARGS; } - if (!ParamCheckUtils::CheckStoreParameter("Relational_default_id", appId_, userId_) || path.empty()) { - callback(INVALID_ARGS, nullptr); - return; + if (!ParamCheckUtils::CheckStoreParameter(storeId, appId_, userId_) || path.empty()) { + return INVALID_ARGS; } - DBProperties properties; - InitStoreProp(option, path, appId_, userId_, properties); + RelationalDBProperties properties; + InitStoreProp(canonicalDir, appId_, userId_, storeId, properties); int errCode = E_OK; auto *conn = GetOneConnectionWithRetry(properties, errCode); - DBStatus status = TransferDBErrno(errCode); if (conn == nullptr) { - callback(status, nullptr); - return; + return TransferDBErrno(errCode); } - auto store = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); - if (store == nullptr) { + delegate = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); + if (delegate == nullptr) { conn->Close(); - callback(DB_ERROR, nullptr); - return; + return DB_ERROR; } - - (void)conn->TriggerAutoSync(); - callback(OK, store); + return OK; } DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *store) @@ -177,5 +172,18 @@ DBStatus RelationalStoreManager::DeleteStore(const std::string &path) LOGE("Delete the kv store error:%d", errCode); return TransferDBErrno(errCode); } + +void RelationalStoreManager::SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) +{ +} + +std::string RelationalStoreManager::GetRelationalStoreIdentifier(const std::string &userId, const std::string &appId, + const std::string &storeId) +{ + if (!ParamCheckUtils::CheckStoreParameter(storeId, appId, userId)) { + return ""; + } + return DBCommon::TransferHashString(userId + "-" + appId + "-" + storeId); +} } // namespace DistributedDB #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h index 37ef57ada6909c8456de6080e71f83d7cdc63a29..e9e6562b4c671d80676734d5f2d97977c8120817 100644 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -19,7 +19,7 @@ #include "relational_db_sync_interface.h" #include "sqlite_single_relational_storage_engine.h" -#include "sqlite_single_ver_continue_token.h" +#include "sqlite_single_ver_relational_continue_token.h" namespace DistributedDB { class RelationalSyncAbleStorage : public RelationalDBSyncInterface, public virtual RefObject { @@ -67,9 +67,6 @@ public: int GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; - // Release the continue token of getting data. - void ReleaseContinueToken(ContinueToken &continueStmtToken) const override; - int PutSyncDataWithQuery(const QueryObject &object, const std::vector &entries, const DeviceID &deviceName) override; @@ -112,7 +109,7 @@ private: int SetMaxTimeStamp(TimeStamp timestamp); // get - int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerContinueToken *&continueStmtToken, + int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerRelationalContinueToken *&token, const DataSizeSpecInfo &dataSizeInfo) const; // put diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h new file mode 100644 index 0000000000000000000000000000000000000000..215745a8eda24467da1c35cb9622fc367753dce9 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/db_properties.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DB_PROPERTIES_H +#define DB_PROPERTIES_H + +#include +#include +#include + +namespace DistributedDB { +class DBProperties { +public: + // Get the string property according the name + std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; + + // Set the string property for the name + void SetStringProp(const std::string &name, const std::string &value); + + // Get the bool property according the name + bool GetBoolProp(const std::string &name, bool defaultValue) const; + + // Set the bool property for the name + void SetBoolProp(const std::string &name, bool value); + + // Get the bool property according the name + int GetIntProp(const std::string &name, int defaultValue) const; + + // Set the integer property for the name + void SetIntProp(const std::string &name, int value); + + // Set all indentifers + void SetIdentifier(const std::string &userId, const std::string &appId, const std::string &storeId); + + static const std::string CREATE_IF_NECESSARY; + static const std::string DATABASE_TYPE; + static const std::string DATA_DIR; + static const std::string USER_ID; + static const std::string APP_ID; + static const std::string STORE_ID; + static const std::string IDENTIFIER_DATA; + static const std::string IDENTIFIER_DIR; + +protected: + DBProperties() = default; + virtual ~DBProperties() = default; + + std::map stringProperties_; + std::map boolProperties_; + std::map intProperties_; +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h index 375823e39b9a306f1d624e693814b234ca52fda6..a6925c11d8662d22abbf9ea7e98951fd1318b1f3 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h @@ -22,17 +22,6 @@ #include "kvdb_properties.h" namespace DistributedDB { -struct SyncTimeRange { - TimeStamp beginTime = 0; - TimeStamp deleteBeginTime = 0; - TimeStamp endTime = static_cast(INT64_MAX); - TimeStamp deleteEndTime = static_cast(INT64_MAX); - bool IsValid() const - { - return (beginTime <= endTime && deleteBeginTime <= deleteEndTime); - } -}; - class ISyncInterface { public: enum { diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h index 6337094af09f61f30f7e164874fea1a525700b92..c985b0d832154b80f63d72c7d10f5298b3c4917f 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h @@ -20,41 +20,14 @@ #include #include +#include "db_properties.h" #include "schema_object.h" namespace DistributedDB { -class DBProperties { -public: - DBProperties() = default; - ~DBProperties() = default; - - // Get the string property according the name - std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; - - // Set the string property for the name - void SetStringProp(const std::string &name, const std::string &value); - - // Get the bool property according the name - bool GetBoolProp(const std::string &name, bool defaultValue) const; - - // Set the bool property for the name - void SetBoolProp(const std::string &name, bool value); - - // Get the bool property according the name - int GetIntProp(const std::string &name, int defaultValue) const; - - // Set the integer property for the name - void SetIntProp(const std::string &name, int value); -protected: - std::map stringProperties_; - std::map boolProperties_; - std::map intProperties_; -}; - class KvDBProperties final : public DBProperties { public: KvDBProperties(); - ~KvDBProperties(); + ~KvDBProperties() override; // Get the sub directory for different type database. static std::string GetStoreSubDirectory(int type); @@ -83,18 +56,10 @@ public: // The upper code will not change the schema if it is already set const SchemaObject &GetSchemaConstRef() const; - static const std::string CREATE_IF_NECESSARY; - static const std::string DATABASE_TYPE; - static const std::string DATA_DIR; - static const std::string USER_ID; - static const std::string APP_ID; - static const std::string STORE_ID; static const std::string FILE_NAME; static const std::string SYNC_MODE; static const std::string MEMORY_MODE; static const std::string ENCRYPTED_MODE; - static const std::string IDENTIFIER_DATA; - static const std::string IDENTIFIER_DIR; static const std::string FIRST_OPEN_IS_READ_ONLY; static const std::string CREATE_DIR_BY_STORE_ID_ONLY; static const std::string SECURITY_LABEL; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h index e502357a79c436dd4b25aea0853765289aa27cde..7b84fc277e90855bc36ec9cb25f615108b8a2752 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -16,10 +16,12 @@ #define RELATIONAL_STORE_CONNECTION_H #ifdef RELATIONAL_STORE +#include "relational_store_delegate.h" #include #include + +#include "db_types.h" #include "macro_utils.h" -#include "relational_store_delegate.h" #include "ref_object.h" namespace DistributedDB { @@ -30,15 +32,14 @@ public: struct SyncInfo { const std::vector &devices; SyncMode mode = SYNC_MODE_PUSH_PULL; - SyncStatusCallback &onComplete; + const SyncStatusCallback &onComplete; const Query &query; bool wait = true; }; - RelationalStoreConnection() = default; - explicit RelationalStoreConnection(IRelationalStore *store) - { - store_ = store; - }; + + RelationalStoreConnection(); + + explicit RelationalStoreConnection(IRelationalStore *store); virtual ~RelationalStoreConnection() = default; @@ -49,8 +50,8 @@ public: virtual int TriggerAutoSync() = 0; virtual int SyncToDevice(SyncInfo &info) = 0; virtual std::string GetIdentifier() = 0; - virtual int CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) = 0; + virtual int CreateDistributedTable(const std::string &tableName) = 0; + virtual int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) = 0; protected: // Get the stashed 'KvDB_ pointer' without ref. @@ -62,7 +63,7 @@ protected: virtual int Pragma(int cmd, void *parameter); IRelationalStore *store_ = nullptr; - std::atomic isExclusive_ = false; + std::atomic isExclusive_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h similarity index 53% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h rename to services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h index 23ad7d58a97e917e0510cf5b26ebda1bb965e2d0..4877e7220343a4d11111b67925624b4b7ed902c1 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relationaldb_properties.h @@ -12,17 +12,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H -#define MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H +#ifndef RELATIONALDB_PROPERTIES_H +#define RELATIONALDB_PROPERTIES_H -#include -#include "single_ver_sync_task_context.h" +#include +#include +#include + +#include "db_properties.h" +#include "relational_schema_object.h" namespace DistributedDB { -class MockSingleVerSyncTaskContext : public SingleVerSyncTaskContext { +class RelationalDBProperties final : public DBProperties { public: - MOCK_METHOD1(HandleDataRequestRecv, int(const Message *)); - MOCK_CONST_METHOD0(IsReceiveWaterMarkErr, bool(void)); + RelationalDBProperties(); + ~RelationalDBProperties() override; + + // is schema exist + bool IsSchemaExist() const; + + // set schema + void SetSchema(const RelationalSchemaObject &schema); + + // get schema + RelationalSchemaObject GetSchema() const; + +private: + RelationalSchemaObject schema_; }; -} // namespace DistributedDB -#endif // MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp index c4e020261e3acc597b205257197520cb772ff2a5..a060a8279fc8c81192972afbce2458bb4d76e122 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -118,17 +118,18 @@ uint32_t DataTransformer::CalDataValueLength(const DataValue &dataValue) return 0u; } uint32_t length = 0; - std::string str; switch (dataValue.GetType()) { case StorageType::STORAGE_TYPE_BLOB: (void)dataValue.GetBlobLength(length); length = Parcel::GetEightByteAlign(length); length += Parcel::GetUInt32Len(); // record data length break; - case StorageType::STORAGE_TYPE_TEXT: + case StorageType::STORAGE_TYPE_TEXT: { + std::string str; (void)dataValue.GetText(str); length = Parcel::GetStringLen(str); break; + } default: break; } @@ -245,7 +246,7 @@ int SerializeBlobValue(const DataValue &dataValue, Parcel &parcel) { Blob val; (void)dataValue.GetBlob(val); - const uint32_t &size = val.GetSize(); + uint32_t size = val.GetSize(); if (size == 0) { return SerializeNullValue(dataValue, parcel); } @@ -301,27 +302,22 @@ int DataTransformer::SerializeValue(Value &value, const RowData &rowData, const uint32_t totalLength = Parcel::GetUInt64Len(); // first record field count for (uint32_t i = 0; i < rowData.size(); ++i) { const auto &dataValue = rowData[i]; - const auto &fieldInfo = fieldInfoList[i]; - if (dataValue.GetType() != StorageType::STORAGE_TYPE_NULL && - dataValue.GetType() != fieldInfo.GetStorageType()) { - return -E_INVALID_DATA; - } if (typeFuncMap.find(dataValue.GetType()) == typeFuncMap.end()) { return -E_NOT_SUPPORT; } + totalLength += Parcel::GetUInt32Len(); // For save the dataValue's type. uint32_t dataLength = CalDataValueLength(dataValue); totalLength += dataLength; } value.resize(totalLength); + if (value.size() != totalLength) { + return -E_OUT_OF_MEMORY; + } Parcel parcel(value.data(), value.size()); (void)parcel.WriteUInt64(rowData.size()); - int index = 0; for (const auto &dataValue : rowData) { - const auto &fieldInfo = fieldInfoList[index++]; StorageType type = dataValue.GetType(); - if (dataValue.GetType() == StorageType::STORAGE_TYPE_NULL) { - type = fieldInfo.GetStorageType(); - } + parcel.WriteUInt32(static_cast(type)); int errCode = typeFuncMap[type].serializeFunc(dataValue, parcel); if (errCode != E_OK) { value.clear(); @@ -346,7 +342,13 @@ int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalDa DataValue dataValue; LOGD("[DataTransformer][DeSerializeValue] start deSerialize %s type %d", fieldInfo.GetFieldName().c_str(), fieldInfo.GetStorageType()); - int errCode = typeFuncMap[fieldInfo.GetStorageType()].deSerializeFunc(dataValue, parcel); + uint32_t type = 0; + parcel.ReadUInt32(type); + auto iter = typeFuncMap.find(static_cast(type)); + if (iter == typeFuncMap.end()) { + return -E_PARSE_FAIL; + } + int errCode = iter->second.deSerializeFunc(dataValue, parcel); if (errCode != E_OK) { LOGD("[DataTransformer][DeSerializeValue] deSerialize %s failed", fieldInfo.GetFieldName().c_str()); return errCode; @@ -361,21 +363,20 @@ int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalDa if ((uint32_t)index >= valueList.size()) { return -E_INTERNAL_ERROR; // should not happen } - if (valueList[index].GetType() == StorageType::STORAGE_TYPE_NULL) { - optionalData.push_back(std::nullopt); - } else { - optionalData.push_back(valueList[index]); - } + optionalData.push_back(valueList[index]); } return E_OK; } int DataTransformer::SerializeHashKey(Key &key, const std::string &hashKey) { - key.resize(Parcel::GetStringLen(hashKey), 0); - Parcel parcel(key.data(), Parcel::GetStringLen(hashKey)); - int errCode = parcel.WriteString(hashKey); - return errCode; + uint32_t len = Parcel::GetStringLen(hashKey); + key.resize(len, 0); + if (key.size() != len) { + return -E_OUT_OF_MEMORY; + } + Parcel parcel(key.data(), len); + return parcel.WriteString(hashKey); } int DataTransformer::DeSerializeHashKey(const Key &key, std::string &hashKey) diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d6920dcc8cd16aa64f907a0f63f55e4626acfe58 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/db_properties.cpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "db_common.h" +#include "db_properties.h" + +namespace DistributedDB { +const std::string DBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; +const std::string DBProperties::DATABASE_TYPE = "databaseType"; +const std::string DBProperties::DATA_DIR = "dataDir"; +const std::string DBProperties::USER_ID = "userId"; +const std::string DBProperties::APP_ID = "appId"; +const std::string DBProperties::STORE_ID = "storeId"; +const std::string DBProperties::IDENTIFIER_DATA = "identifier"; +const std::string DBProperties::IDENTIFIER_DIR = "identifierDir"; + +std::string DBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const +{ + auto iter = stringProperties_.find(name); + if (iter != stringProperties_.end()) { + return iter->second; + } + return defaultValue; +} + +void DBProperties::SetStringProp(const std::string &name, const std::string &value) +{ + stringProperties_[name] = value; +} + +bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const +{ + auto iter = boolProperties_.find(name); + if (iter != boolProperties_.end()) { + return iter->second; + } + return defaultValue; +} + +void DBProperties::SetBoolProp(const std::string &name, bool value) +{ + boolProperties_[name] = value; +} + +int DBProperties::GetIntProp(const std::string &name, int defaultValue) const +{ + auto iter = intProperties_.find(name); + if (iter != intProperties_.end()) { + return iter->second; + } + return defaultValue; +} + +void DBProperties::SetIntProp(const std::string &name, int value) +{ + intProperties_[name] = value; +} + +void DBProperties::SetIdentifier(const std::string &userId, const std::string &appId, const std::string &storeId) +{ + SetStringProp(DBProperties::APP_ID, appId); + SetStringProp(DBProperties::USER_ID, userId); + SetStringProp(DBProperties::STORE_ID, storeId); + std::string hashIdentifier = DBCommon::TransferHashString(DBCommon::GenerateIdentifierId(storeId, appId, userId)); + SetStringProp(DBProperties::IDENTIFIER_DATA, hashIdentifier); +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h index 2d8cf45e0f67e7d606522b3f9efb210e5d315e04..a9714e556139794d230c69c96d84cf45007b7a37 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h @@ -20,7 +20,7 @@ #include #include "ref_object.h" -#include "kvdb_properties.h" +#include "relationaldb_properties.h" #include "relational_store_connection.h" namespace DistributedDB { @@ -31,7 +31,7 @@ public: DISABLE_COPY_ASSIGN_MOVE(IRelationalStore); // Open the database. - virtual int Open(const DBProperties &kvDBProp) = 0; + virtual int Open(const RelationalDBProperties &properties) = 0; virtual void WakeUpSyncer() = 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp index ee95611ea9f25a4c0e7e85b499b9a7321a324109..c10569d8213c9230958c4ff4719f5b1e9531942e 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp @@ -18,17 +18,9 @@ #include "db_constant.h" namespace DistributedDB { -const std::string KvDBProperties::CREATE_IF_NECESSARY = "createIfNecessary"; -const std::string KvDBProperties::DATABASE_TYPE = "databaseType"; -const std::string KvDBProperties::DATA_DIR = "dataDir"; -const std::string KvDBProperties::USER_ID = "userId"; -const std::string KvDBProperties::APP_ID = "appId"; -const std::string KvDBProperties::STORE_ID = "storeId"; const std::string KvDBProperties::FILE_NAME = "fileName"; const std::string KvDBProperties::MEMORY_MODE = "memoryMode"; const std::string KvDBProperties::ENCRYPTED_MODE = "isEncryptedDb"; -const std::string KvDBProperties::IDENTIFIER_DATA = "identifier"; -const std::string KvDBProperties::IDENTIFIER_DIR = "identifierDir"; const std::string KvDBProperties::FIRST_OPEN_IS_READ_ONLY = "firstOpenIsReadOnly"; const std::string KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY = "createDirByStoreIdOnly"; const std::string KvDBProperties::SECURITY_LABEL = "securityLabel"; @@ -59,51 +51,6 @@ std::string KvDBProperties::GetStoreSubDirectory(int type) } } -std::string DBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const -{ - auto iter = stringProperties_.find(name); - if (iter != stringProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetStringProp(const std::string &name, const std::string &value) -{ - stringProperties_[name] = value; -} - -bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const -{ - auto iter = boolProperties_.find(name); - if (iter != boolProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetBoolProp(const std::string &name, bool value) -{ - boolProperties_[name] = value; -} - -int DBProperties::GetIntProp(const std::string &name, int defaultValue) const -{ - auto iter = intProperties_.find(name); - if (iter != intProperties_.end()) { - return iter->second; - } else { - return defaultValue; - } -} - -void DBProperties::SetIntProp(const std::string &name, int value) -{ - intProperties_[name] = value; -} - void KvDBProperties::GetPassword(CipherType &type, CipherPassword &password) const { type = cipherType_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp index 4fc241f370cb7f2491f2662f268878c0b745df8b..c04d81193a1bc286808972e6c9134d385766768a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp @@ -18,6 +18,13 @@ #include "sqlite_single_ver_relational_storage_executor.h" namespace DistributedDB { +RelationalStoreConnection::RelationalStoreConnection() : isExclusive_(false) +{} + +RelationalStoreConnection::RelationalStoreConnection(IRelationalStore *store) + : store_(store), isExclusive_(false) +{} + int RelationalStoreConnection::Pragma(int cmd, void *parameter) { return E_OK; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp index e48bc8230e24d145e283748d8e9e40b23f8c1f9c..366d78863fcc80e252f65c8a39afbdd356227c71 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp @@ -18,8 +18,9 @@ #include #include -#include "sqlite_relational_store.h" +#include "db_common.h" #include "db_errno.h" +#include "sqlite_relational_store.h" #include "log_print.h" namespace DistributedDB { @@ -54,10 +55,10 @@ int RelationalStoreInstance::CheckDatabaseFileStatus(const std::string &id) return E_OK; } -static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCode) +static IRelationalStore *GetFromCache(const RelationalDBProperties &properties, int &errCode) { errCode = E_OK; - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); auto iter = dbs_.find(identifier); if (iter == dbs_.end()) { errCode = -E_NOT_FOUND; @@ -65,7 +66,6 @@ static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCo } auto *db = iter->second; - if (db == nullptr) { LOGE("Store cache is nullptr, there may be a logic error"); errCode = -E_INTERNAL_ERROR; @@ -76,21 +76,21 @@ static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCo } // Save to IKvDB to the global map -void RelationalStoreInstance::RemoveKvDBFromCache(const DBProperties &properties) +void RelationalStoreInstance::RemoveKvDBFromCache(const RelationalDBProperties &properties) { std::lock_guard lockGuard(storeLock_); - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); dbs_.erase(identifier); } -void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties) +void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const RelationalDBProperties &properties) { if (store == nullptr) { return; } { - std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifier = properties.GetStringProp(RelationalDBProperties::IDENTIFIER_DATA, ""); store->WakeUpSyncer(); if (dbs_.count(identifier) == 0) { dbs_.insert(std::pair(identifier, store)); @@ -98,11 +98,11 @@ void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const DBP } } -IRelationalStore *RelationalStoreInstance::OpenDatabase(const DBProperties &properties, int &errCode) +IRelationalStore *RelationalStoreInstance::OpenDatabase(const RelationalDBProperties &properties, int &errCode) { auto db = new (std::nothrow) SQLiteRelationalStore(); if (db == nullptr) { - LOGE("Failed to get IKvDB! err:%d", errCode); + LOGE("Failed to get relational store! err:%d", errCode); return nullptr; } @@ -122,7 +122,7 @@ IRelationalStore *RelationalStoreInstance::OpenDatabase(const DBProperties &prop return db; } -IRelationalStore *RelationalStoreInstance::GetDataBase(const DBProperties &properties, int &errCode) +IRelationalStore *RelationalStoreInstance::GetDataBase(const RelationalDBProperties &properties, int &errCode) { std::lock_guard lockGuard(storeLock_); auto *db = GetFromCache(properties, errCode); @@ -145,8 +145,11 @@ IRelationalStore *RelationalStoreInstance::GetDataBase(const DBProperties &prope return db; } -RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const DBProperties &properties, int &errCode) +RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const RelationalDBProperties &properties, + int &errCode) { + std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + LOGD("Begin to get [%s] database connection.", STR_MASK(DBCommon::TransferStringToHex(identifier))); IRelationalStore *db = GetDataBase(properties, errCode); if (db == nullptr) { LOGE("Failed to open the db:%d", errCode); @@ -157,9 +160,8 @@ RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const if (connection == nullptr) { // not kill db, Other operations like import may be used concurrently LOGE("Failed to get the db connect for delegate:%d", errCode); } - RefObject::DecObjRef(db); // restore the reference increased by the cache. - // kvDB = nullptr; + RefObject::DecObjRef(db); // restore the reference increased by the cache. return connection; } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp index c8082bad6ac946c1421d1f5bd24f224a3bcc3ec2..2933959e9ce4153852a8180ef9f06ed1f2e7c20a 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -24,6 +24,10 @@ namespace DistributedDB { } \ } while (0) +namespace { +static constexpr float QUERY_SYNC_THRESHOLD = 0.50; +} + RelationalSyncAbleStorage::RelationalSyncAbleStorage(StorageEngine *engine) : storageEngine_(static_cast(engine)) {} @@ -196,15 +200,6 @@ const KvDBProperties &RelationalSyncAbleStorage::GetDbProperties() const return properties; } -static void ReleaseKvEntries(std::vector &entries) -{ - for (auto &itemEntry : entries) { - delete itemEntry; - itemEntry = nullptr; - } - entries.clear(); -} - static int GetKvEntriesByDataItems(std::vector &entries, std::vector &dataItems) { int errCode = E_OK; @@ -213,7 +208,7 @@ static int GetKvEntriesByDataItems(std::vector &entries, std if (entry == nullptr) { errCode = -E_OUT_OF_MEMORY; LOGE("GetKvEntries failed, errCode:%d", errCode); - ReleaseKvEntries(entries); + SingleVerKvEntry::Release(entries); break; } entry->SetEntryData(std::move(item)); @@ -233,11 +228,10 @@ static size_t GetDataItemSerialSize(const DataItem &item, size_t appendLen) return dataSize; } -static constexpr float QUERY_SYNC_THRESHOLD = 0.50; static bool CanHoldDeletedData(const std::vector &dataItems, const DataSizeSpecInfo &dataSizeInfo, size_t appendLen) { - bool reachThreshold = false; + bool reachThreshold = (dataItems.size() >= dataSizeInfo.packetSize); for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { blockSize += GetDataItemSerialSize(dataItems[i], appendLen); reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); @@ -246,7 +240,7 @@ static bool CanHoldDeletedData(const std::vector &dataItems, const Dat } static void ProcessContinueTokenForQuerySync(const std::vector &dataItems, int &errCode, - SQLiteSingleVerContinueToken *&token) + SQLiteSingleVerRelationalContinueToken *&token) { if (errCode != -E_UNFINISHED) { // Error happened or get data finished. Token should be cleared. delete token; @@ -261,26 +255,15 @@ static void ProcessContinueTokenForQuerySync(const std::vector &dataIt token = nullptr; return; } - - TimeStamp nextBeginTime = dataItems.back().timeStamp + 1; - if (nextBeginTime > INT64_MAX) { - nextBeginTime = INT64_MAX; - } - bool getDeleteData = ((dataItems.back().flag & DataItem::DELETE_FLAG) != 0); - if (getDeleteData) { - token->FinishGetQueryData(); - token->SetDeletedNextBeginTime("", nextBeginTime); - } else { - token->SetNextBeginTime("", nextBeginTime); - } + token->SetNextBeginTime(dataItems.back()); } /** - * Caller must ensure that parameter continueStmtToken is valid. + * Caller must ensure that parameter token is valid. * If error happened, token will be deleted here. */ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &dataItems, - SQLiteSingleVerContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const + SQLiteSingleVerRelationalContinueToken *&token, const DataSizeSpecInfo &dataSizeInfo) const { if (storageEngine_ == nullptr) { return -E_INVALID_DB; @@ -293,47 +276,29 @@ int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &da goto ERROR; } - errCode = handle->SetTableInfo(continueStmtToken->GetQuery()); + errCode = handle->SetTableInfo(token->GetQuery()); if (errCode != E_OK) { goto ERROR; } - // Get query data. - if (!continueStmtToken->IsGetQueryDataFinished()) { - LOGD("[SingleVerNStore] Get query data between %llu and %llu.", continueStmtToken->GetQueryBeginTime(), - continueStmtToken->GetQueryEndTime()); - errCode = handle->GetSyncDataByQuery( - dataItems, Parcel::GetAppendedLen(), continueStmtToken->GetQuery(), dataSizeInfo, - // need modify - std::make_pair(continueStmtToken->GetQueryBeginTime(), continueStmtToken->GetQueryEndTime())); - } - - // Get query data finished. - if (errCode == E_OK || errCode == -E_FINISHED) { - // Clear query timeRange. - continueStmtToken->FinishGetQueryData(); - - // Get delete time next. - if (!continueStmtToken->IsGetDeletedDataFinished() && - CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())) { - LOGD("[SingleVerNStore] Get deleted data between %llu and %llu.", continueStmtToken->GetDeletedBeginTime(), - continueStmtToken->GetDeletedEndTime()); - errCode = handle->GetDeletedSyncDataByTimestamp( - dataItems, Parcel::GetAppendedLen(), - // need modify - continueStmtToken->GetDeletedBeginTime(), continueStmtToken->GetDeletedEndTime(), dataSizeInfo); + do { + errCode = handle->GetSyncDataByQuery(dataItems, + Parcel::GetAppendedLen(), + dataSizeInfo, + std::bind(&SQLiteSingleVerRelationalContinueToken::GetStatement, *token, + std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + if (errCode == -E_FINISHED) { + token->FinishGetData(); + errCode = token->IsGetAllDataFinished() ? E_OK : -E_UNFINISHED; } - } - - if (errCode == -E_FINISHED) { - errCode = E_OK; - } + } while (errCode == -E_UNFINISHED && + CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())); ERROR: if (errCode != -E_UNFINISHED && errCode != E_OK) { // Error happened. dataItems.clear(); } - ProcessContinueTokenForQuerySync(dataItems, errCode, continueStmtToken); + ProcessContinueTokenForQuerySync(dataItems, errCode, token); ReleaseHandle(handle); return errCode; } @@ -348,13 +313,24 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return -E_INVALID_ARGS; } - auto token = new (std::nothrow) SQLiteSingleVerContinueToken(timeRange, query); // release in sync module + auto token = new (std::nothrow) SQLiteSingleVerRelationalContinueToken(timeRange, query); if (token == nullptr) { LOGE("[SingleVerNStore] Allocate continue token failed."); return -E_OUT_OF_MEMORY; } - int innerCode; + continueStmtToken = static_cast(token); + return GetSyncDataNext(entries, continueStmtToken, dataSizeInfo); +} + +int RelationalSyncAbleStorage::GetSyncDataNext(std::vector &entries, + ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const +{ + auto token = static_cast(continueStmtToken); + if (!token->CheckValid()) { + return -E_INVALID_ARGS; + } + std::vector dataItems; int errCode = GetSyncDataForQuerySync(dataItems, token, dataSizeInfo); if (errCode != E_OK && errCode != -E_UNFINISHED) { // The code need be sent to outside except new error happened. @@ -362,7 +338,7 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return errCode; } - innerCode = GetKvEntriesByDataItems(entries, dataItems); + int innerCode = GetKvEntriesByDataItems(entries, dataItems); if (innerCode != E_OK) { errCode = innerCode; delete token; @@ -372,18 +348,6 @@ int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRan return errCode; } -int RelationalSyncAbleStorage::GetSyncDataNext(std::vector &entries, - ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const -{ - return E_OK; -} - -// Release the continue token of getting data. -void RelationalSyncAbleStorage::ReleaseContinueToken(ContinueToken &continueStmtToken) const -{ - return; -} - int RelationalSyncAbleStorage::PutSyncDataWithQuery(const QueryObject &object, const std::vector &entries, const DeviceID &deviceName) { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp similarity index 59% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h rename to services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp index 5bd453f601979ee076bf2b2f9c2135c3ba088225..3a87aafb5e5a8a7d829fdc8b3819b806409656e8 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relationaldb_properties.cpp @@ -12,17 +12,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MOCK_SINGLE_VER_DATA_SYNC_H -#define MOCK_SINGLE_VER_DATA_SYNC_H -#include -#include "single_ver_data_sync.h" +#include "relationaldb_properties.h" namespace DistributedDB { -class MockSingleVerDataSync : public SingleVerDataSync { -public: - MOCK_METHOD4(SendDataAck, int(SingleVerSyncTaskContext *, const Message *, int32_t, WaterMark)); -}; -} // namespace DistributedDB +RelationalDBProperties::RelationalDBProperties() +{} -#endif // MOCK_SINGLE_VER_DATA_SYNC_H +RelationalDBProperties::~RelationalDBProperties() +{} + +bool RelationalDBProperties::IsSchemaExist() const +{ + return schema_.IsSchemaValid(); +} + +void RelationalDBProperties::SetSchema(const RelationalSchemaObject &schema) +{ + schema_ = schema; +} + +RelationalSchemaObject RelationalDBProperties::GetSchema() const +{ + return schema_; +} +} diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp index ed7bdb058df387637c4cb5f97ec47881e8c10d46..c33c73acd8791ab4afe373ab5f510578e2d80f9d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -15,18 +15,28 @@ #ifdef RELATIONAL_STORE #include "sqlite_relational_store.h" +#include "db_common.h" #include "db_errno.h" #include "log_print.h" #include "db_types.h" #include "sqlite_relational_store_connection.h" +#include "storage_engine_manager.h" namespace DistributedDB { +namespace { + constexpr const char *RELATIONAL_SCHEMA_KEY = "relational_schema"; + constexpr const char *LOG_TABLE_VERSION_KEY = "log_table_versoin"; + constexpr const char *LOG_TABLE_VERSION_1 = "1.0"; +} + SQLiteRelationalStore::~SQLiteRelationalStore() { - delete sqliteStorageEngine_; + if (sqliteStorageEngine_ != nullptr) { + delete sqliteStorageEngine_; + sqliteStorageEngine_ = nullptr; + } } - // Called when a new connection created. void SQLiteRelationalStore::IncreaseConnectionCounter() { @@ -40,7 +50,6 @@ RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) { std::lock_guard lock(connectMutex_); RelationalStoreConnection* connection = new (std::nothrow) SQLiteRelationalStoreConnection(this); - if (connection == nullptr) { errCode = -E_OUT_OF_MEMORY; return nullptr; @@ -50,16 +59,16 @@ RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) return connection; } -static void InitDataBaseOption(const DBProperties &kvDBProp, OpenDbProperties &option) +static void InitDataBaseOption(const RelationalDBProperties &properties, OpenDbProperties &option) { - option.uri = kvDBProp.GetStringProp(KvDBProperties::DATA_DIR, ""); - option.createIfNecessary = kvDBProp.GetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, false); + option.uri = properties.GetStringProp(DBProperties::DATA_DIR, ""); + option.createIfNecessary = properties.GetBoolProp(DBProperties::CREATE_IF_NECESSARY, false); } -int SQLiteRelationalStore::InitStorageEngine(const DBProperties &kvDBProp) +int SQLiteRelationalStore::InitStorageEngine(const RelationalDBProperties &properties) { OpenDbProperties option; - InitDataBaseOption(kvDBProp, option); + InitDataBaseOption(properties, option); StorageEngineAttr poolSize = {1, 1, 0, 16}; // at most 1 write 16 read. int errCode = sqliteStorageEngine_->InitSQLiteStorageEngine(poolSize, option); @@ -69,24 +78,139 @@ int SQLiteRelationalStore::InitStorageEngine(const DBProperties &kvDBProp) return errCode; } -int SQLiteRelationalStore::Open(const DBProperties &properties) +void SQLiteRelationalStore::ReleaseResources() { - sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); - if (sqliteStorageEngine_ == nullptr) { - LOGE("[RelationalStore] Create storage engine failed"); - return -E_OUT_OF_MEMORY; + if (sqliteStorageEngine_ != nullptr) { + sqliteStorageEngine_->ClearEnginePasswd(); + (void)StorageEngineManager::ReleaseStorageEngine(sqliteStorageEngine_); } +} - int errCode = InitStorageEngine(properties); +int SQLiteRelationalStore::CheckDBMode() +{ + int errCode = E_OK; + auto *handle = GetHandle(false, errCode); + if (handle == nullptr) { + return errCode; + } + errCode = handle->CheckDBModeForRelational(); if (errCode != E_OK) { - LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); + LOGE("check relational DB mode failed. %d", errCode); + } + + ReleaseHandle(handle); + return errCode; +} + +int SQLiteRelationalStore::GetSchemaFromMeta() +{ + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + int errCode = storageEngine_->GetMetaData(schemaKey, schemaVal); + if (errCode != E_OK && errCode != -E_NOT_FOUND) { + LOGE("Get relationale schema from meta table failed. %d", errCode); + return errCode; + } else if (errCode == -E_NOT_FOUND) { + LOGW("No relational schema info was found."); + return E_OK; + } + + std::string schemaStr; + DBCommon::VectorToString(schemaVal, schemaStr); + RelationalSchemaObject schema; + errCode = schema.ParseFromSchemaString(schemaStr); + if (errCode != E_OK) { + LOGE("Parse schema string from mata table failed."); return errCode; } - storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); - syncEngine_ = std::make_shared(storageEngine_); + + std::lock_guard lock(schemaMutex_); + properties_.SetSchema(schema); return E_OK; } +int SQLiteRelationalStore::SaveSchemaToMeta() +{ + const Key schemaKey(RELATIONAL_SCHEMA_KEY, RELATIONAL_SCHEMA_KEY + strlen(RELATIONAL_SCHEMA_KEY)); + Value schemaVal; + DBCommon::StringToVector(properties_.GetSchema().ToSchemaString(), schemaVal); + int errCode = storageEngine_->PutMetaData(schemaKey, schemaVal); + if (errCode != E_OK) { + LOGE("Save relational schema to meta table failed. %d", errCode); + } + return errCode; +} + +int SQLiteRelationalStore::SaveLogTableVersionToMeta() +{ + LOGD("save log table version to meta table, key: %s, val: %s", LOG_TABLE_VERSION_KEY, LOG_TABLE_VERSION_1); + return E_OK; +} + +int SQLiteRelationalStore::CleanDistributedDeviceTable() +{ + return E_OK; +} + +int SQLiteRelationalStore::Open(const RelationalDBProperties &properties) +{ + std::lock_guard lock(initalMutex_); + if (isInitialized_) { + LOGD("[RelationalStore][Open] relational db was already inited."); + return E_OK; + } + + sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); + if (sqliteStorageEngine_ == nullptr) { + LOGE("[RelationalStore][Open] Create storage engine failed"); + return -E_OUT_OF_MEMORY; + } + + int errCode = E_OK; + do { + errCode = InitStorageEngine(properties); + if (errCode != E_OK) { + LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); + break; + } + + storageEngine_ = new (std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); + if (storageEngine_ == nullptr) { + LOGE("[RelationalStore][Open] Create syncable storage failed"); + errCode = -E_OUT_OF_MEMORY; + break; + } + + errCode = CheckDBMode(); + if (errCode != E_OK) { + break; + } + + properties_ = properties; + errCode = GetSchemaFromMeta(); + if (errCode != E_OK) { + break; + } + + errCode = SaveLogTableVersionToMeta(); + if (errCode != E_OK) { + break; + } + + errCode = CleanDistributedDeviceTable(); + if (errCode != E_OK) { + break; + } + + syncEngine_ = std::make_shared(storageEngine_); + isInitialized_ = true; + return E_OK; + } while (false); + + ReleaseResources(); + return errCode; +} + void SQLiteRelationalStore::OnClose(const std::function ¬ifier) { AutoLock lockGuard(this); @@ -179,5 +303,49 @@ void SQLiteRelationalStore::WakeUpSyncer() { syncEngine_->WakeUpSyncer(); } + + +int SQLiteRelationalStore::CreateDistributedTable(const std::string &tableName) +{ + int errCode = E_OK; + std::lock_guard lock(schemaMutex_); + auto schema = properties_.GetSchema(); + if (schema.GetTable(tableName).GetTableName() == tableName) { + LOGW("distributed table was already created."); + return E_OK; + } + + if (schema.GetTables().size() >= DBConstant::MAX_DISTRIBUTED_TABLE_COUNT) { + LOGW("The number of distributed tables is exceeds limit."); + return -E_MAX_LIMITS; + } + + LOGD("Create distributed table."); + auto *handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + + TableInfo table; + errCode = handle->CreateDistributedTable(tableName, table); + if (errCode != E_OK) { + LOGE("create distributed table failed. %d", errCode); + (void)handle->Rollback(); + ReleaseHandle(handle); + return errCode; + } + schema.AddRelationalTable(table); + properties_.SetSchema(schema); + (void)handle->Commit(); + + ReleaseHandle(handle); + return SaveSchemaToMeta(); +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h index f41dd0839819f9e02d4a1933561241f3121d9789..dd0b50360eebc1fbfccf023a7164d52031495f3c 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -33,7 +33,7 @@ public: ~SQLiteRelationalStore() override; RelationalStoreConnection *GetDBConnection(int &errCode) override; - int Open(const DBProperties &properties) override; + int Open(const RelationalDBProperties &properties) override; void OnClose(const std::function ¬ifier); SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode) const; @@ -43,16 +43,27 @@ public: void ReleaseDBConnection(RelationalStoreConnection *connection); - void WakeUpSyncer(); + void WakeUpSyncer() override; // for test mock const RelationalSyncAbleStorage *GetStorageEngine() { return storageEngine_; } + + int CreateDistributedTable(const std::string &tableName); private: + void ReleaseResources(); + // 1 store 1 connection void DecreaseConnectionCounter(); + int CheckDBMode(); + int GetSchemaFromMeta(); + int SaveSchemaToMeta(); + + int SaveLogTableVersionToMeta(); + + int CleanDistributedDeviceTable(); // use for sync Interactive std::shared_ptr syncEngine_ = nullptr; // For storage operate sync function @@ -61,10 +72,17 @@ private: SQLiteSingleRelationalStorageEngine *sqliteStorageEngine_ = nullptr; void IncreaseConnectionCounter(); - int InitStorageEngine(const DBProperties &kvDBProp); + int InitStorageEngine(const RelationalDBProperties &kvDBProp); std::mutex connectMutex_; std::atomic connectionCount_ = 0; std::vector> closeNotifiers_; + + RelationalDBProperties properties_; + + mutable std::mutex initalMutex_; + bool isInitialized_ = false; + + mutable std::mutex schemaMutex_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp index d322575d709847e339e745318ea1d06e5b383aff..de529540bcc2f106bd9f018634fbc82fa80bfcb5 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -58,7 +58,7 @@ SQLiteSingleVerRelationalStorageExecutor *SQLiteRelationalStoreConnection::GetEx auto *store = GetDB(); if (store == nullptr) { errCode = -E_NOT_INIT; - LOGE("[SingleVerConnection] store is null, get executor failed! errCode = [%d]", errCode); + LOGE("[RelationalConnection] store is null, get executor failed! errCode = [%d]", errCode); return nullptr; } @@ -93,7 +93,7 @@ int SQLiteRelationalStoreConnection::StartTransaction() return errCode; } - LOGD("[SingleVerConnection] Start transaction finish."); + LOGD("[RelationalConnection] Start transaction finish."); writeHandle_ = handle; transactingFlag_.store(true); return E_OK; @@ -135,20 +135,17 @@ int SQLiteRelationalStoreConnection::RollBack() return errCode; } -int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) +int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &tableName) { - int errCode = StartTransaction(); - if (errCode != E_OK && errCode != E_TRANSACT_STATE) { - return errCode; + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[RelationalConnection] store is null, get DB failed!"); + return -E_INVALID_CONNECTION; } - errCode = writeHandle_->CreateDistributedTable(tableName, option); - if (errCode == E_OK) { - errCode = Commit(); - } else { - int innerCode = RollBack(); - errCode = (innerCode != E_OK) ? innerCode : errCode; + int errCode = store->CreateDistributedTable(tableName); + if (errCode != E_OK) { + LOGE("[RelationalConnection] crete distributed table failed. %d", errCode); } return errCode; } @@ -167,7 +164,7 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) { auto *store = GetDB(); if (store == nullptr) { - LOGE("[SingleVerConnection] store is null, get executor failed!"); + LOGE("[RelationalConnection] store is null, get executor failed!"); return -E_INVALID_CONNECTION; } @@ -190,5 +187,10 @@ int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) } return E_OK; } + +int SQLiteRelationalStoreConnection::RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) +{ + return E_OK; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h index 42184112fc0917c0a31fdbf19fc1e440db58044c..552447e76e6ab4e6aae8ae70aa84d35df375cfe4 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -37,12 +37,12 @@ public: int TriggerAutoSync() override; int SyncToDevice(SyncInfo &info) override; std::string GetIdentifier() override; - int CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) override; + int CreateDistributedTable(const std::string &tableName) override; + int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) override; protected: - int Pragma(int cmd, void *parameter); + int Pragma(int cmd, void *parameter) override; private: SQLiteSingleVerRelationalStorageExecutor *GetExecutor(bool isWrite, int &errCode) const; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp index ab12fd5d85fe01e9b000b2c3c335c35299da09ba..6ef4b916eda3dd8d12d30783bbbaeff0ca1ec604 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -52,12 +52,12 @@ int SQLiteSingleRelationalStorageEngine::RegisterFunction(sqlite3 *db) const int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, StorageExecutor *&handle) { sqlite3 *db = nullptr; - int errCode = SQLiteUtils::OpenDatabase(option_, db); + int errCode = SQLiteUtils::OpenDatabase(option_, db, false); if (errCode != E_OK) { return errCode; } do { - errCode = Upgrade(db); + errCode = Upgrade(db); // cerate meta_data table. if (errCode != E_OK) { break; } @@ -66,6 +66,7 @@ int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, Storage if (errCode != E_OK) { break; } + handle = NewSQLiteStorageExecutor(db, isWrite, false); if (handle == nullptr) { LOGE("[Relational] New SQLiteStorageExecutor[%d] for the pool failed.", isWrite); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp index 2053fb76b9990f533e89313d0e03b4bcbf4e407b..212b13f301e0b457e2d708a93dabca385c0368a6 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp @@ -83,6 +83,38 @@ std::string FieldValue2String(const FieldValue &val, QueryValueType type) return ""; } } + +std::string GetSelectAndFromClauseForRDB(const std::string &tableName) +{ + return "SELECT b.data_key," + "b.device," + "b.ori_device," + "b.timestamp as " + DBConstant::TIMESTAMP_ALIAS + "," + "b.wtimestamp," + "b.flag," + "b.hash_key," + "a.* " + "FROM " + tableName + " AS a INNER JOIN " + DBConstant::RELATIONAL_PREFIX + tableName + "_log AS b " + "ON a.rowid=b.data_key "; +} + +std::string GetTimeRangeClauseForRDB() +{ + return " AND (" + DBConstant::TIMESTAMP_ALIAS + ">=? AND " + DBConstant::TIMESTAMP_ALIAS + "=? AND " + DBConstant::TIMESTAMP_ALIAS + "=? AND naturalbase_rdb_timestamp=? AND naturalbase_rdb_timestamp>> mulDevTableTimeRange_; - std::map>> mulDevTableDeletedTimeRange_; }; } // namespace DistributedDB #endif // SQLITE_SINGLE_VER_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp new file mode 100644 index 0000000000000000000000000000000000000000..512a9a46e10d3e5f70791e7310b2ba03e767460e --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "sqlite_single_ver_relational_continue_token.h" +#include "sqlite_utils.h" + +namespace DistributedDB { +SQLiteSingleVerRelationalContinueToken::SQLiteSingleVerRelationalContinueToken( + const SyncTimeRange &timeRange, const QueryObject &object) + : isGettingDeletedData_(false), queryObj_(object), tableName_(object.GetTableName()), timeRange_(timeRange) +{} + +bool SQLiteSingleVerRelationalContinueToken::CheckValid() const +{ + bool isValid = (magicBegin_ == MAGIC_BEGIN && magicEnd_ == MAGIC_END); + if (!isValid) { + LOGE("Invalid continue token."); + } + return isValid; +} + +int SQLiteSingleVerRelationalContinueToken::GetStatement(sqlite3 *db, sqlite3_stmt *&stmt, bool &isGettingDeletedData) +{ + isGettingDeletedData = isGettingDeletedData_; + if (isGettingDeletedData) { + return GetDeletedDataStmt(db, stmt); + } + return GetQuerySyncStatement(db, stmt); +} + +void SQLiteSingleVerRelationalContinueToken::SetNextBeginTime(const DataItem &theLastItem) +{ + TimeStamp nextBeginTime = theLastItem.timeStamp + 1; + if (nextBeginTime > INT64_MAX) { + nextBeginTime = INT64_MAX; + } + if (!isGettingDeletedData_) { + timeRange_.beginTime = nextBeginTime; + return; + } + if ((theLastItem.flag & DataItem::DELETE_FLAG) != 0) { // The last one could be non-deleted. + timeRange_.deleteBeginTime = nextBeginTime; + } +} + +void SQLiteSingleVerRelationalContinueToken::FinishGetData() +{ + if (isGettingDeletedData_) { + timeRange_.deleteEndTime = 0; + return; + } + isGettingDeletedData_ = true; + timeRange_.endTime = 0; + return; +} + +bool SQLiteSingleVerRelationalContinueToken::IsGetAllDataFinished() const +{ + return timeRange_.beginTime >= timeRange_.endTime && timeRange_.deleteBeginTime >= timeRange_.deleteEndTime; +} + +int SQLiteSingleVerRelationalContinueToken::GetQuerySyncStatement(sqlite3 *db, sqlite3_stmt *&stmt) +{ + int errCode = E_OK; + SqliteQueryHelper helper = queryObj_.GetQueryHelper(errCode); + if (errCode != E_OK) { + return errCode; + } + return helper.GetRelationalQueryStatement(db, timeRange_.beginTime, timeRange_.endTime, stmt); +} + +int SQLiteSingleVerRelationalContinueToken::GetDeletedDataStmt(sqlite3 *db, sqlite3_stmt *&stmt) const +{ + // get stmt + int errCode = E_OK; + const std::string sql = GetDeletedDataSQL(); + errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + goto ERROR; + } + + // bind stmt + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 1, timeRange_.deleteBeginTime); // 1 means begin time + if (errCode != E_OK) { + goto ERROR; + } + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 2, timeRange_.deleteEndTime); // 2 means end time + if (errCode != E_OK) { + goto ERROR; + } + return errCode; + +ERROR: + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} + +const QueryObject &SQLiteSingleVerRelationalContinueToken::GetQuery() const +{ + return queryObj_; +} + +std::string SQLiteSingleVerRelationalContinueToken::GetDeletedDataSQL() const +{ + std::string tableName = DBConstant::RELATIONAL_PREFIX + tableName_ + "_log"; + return "SELECT * FROM " + tableName + + " WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03 = 0x03) ORDER BY timestamp ASC;"; +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h old mode 100755 new mode 100644 similarity index 31% rename from services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h rename to services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h index 6d77f0ccfc7a011fa5aea8f8d811cb7040223d05..912aa4c960addb2084a038d0d9b51241e709ac82 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_continue_token.h @@ -12,34 +12,44 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#ifndef SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H +#define SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H +#ifdef RELATIONAL_STORE +#include +#include -#ifndef SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H -#define SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H - -#include "sliding_window_receiver.h" -#include "sliding_window_sender.h" +#include "db_types.h" +#include "query_object.h" namespace DistributedDB { -class SingleVerDataSyncWithSlidingWindow { +class SQLiteSingleVerRelationalContinueToken { public: - SingleVerDataSyncWithSlidingWindow() = default; - ~SingleVerDataSyncWithSlidingWindow() = default; - DISABLE_COPY_ASSIGN_MOVE(SingleVerDataSyncWithSlidingWindow); + SQLiteSingleVerRelationalContinueToken(const SyncTimeRange &timeRange, const QueryObject &queryObject); + ~SQLiteSingleVerRelationalContinueToken() = default; - int SenderStart(int32_t mode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int PreHandleSenderAckRecv(const Message *message); - int SenderAckRecv(const Message *message); - void SenderClear(); - void SetSenderErr(bool isErr); + // Check the magic number at the beginning and end of the SQLiteSingleVerRelationalContinueToken. + bool CheckValid() const; + // The statement should be release by the caller. + int GetStatement(sqlite3 *db, sqlite3_stmt *&stmt, bool &isGettingDeletedData); + void SetNextBeginTime(const DataItem &theLastItem); + void FinishGetData(); + bool IsGetAllDataFinished() const; + const QueryObject &GetQuery() const; - int ReceiverInit(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int SenderInit(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - void ReceiverClear(); - int Receive(Message *inMsg); private: - SlidingWindowSender sender_; - SlidingWindowReceiver receiver_; + std::string GetDeletedDataSQL() const; + int GetQuerySyncStatement(sqlite3 *db, sqlite3_stmt *&stmt); + int GetDeletedDataStmt(sqlite3 *db, sqlite3_stmt *&stmt) const; + + static const unsigned int MAGIC_BEGIN = 0x600D0AC7; // for token guard + static const unsigned int MAGIC_END = 0x0AC7600D; // for token guard + unsigned int magicBegin_ = MAGIC_BEGIN; + int isGettingDeletedData_ = false; + QueryObject queryObj_; + const std::string &tableName_; + SyncTimeRange timeRange_; + unsigned int magicEnd_ = MAGIC_END; }; } // namespace DistributedDB -#endif // SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H - +#endif // RELATIONAL_STORE +#endif // SQLITE_SINGLE_VER_RELATIONAL_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp similarity index 68% rename from services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp rename to services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp index 167adfdb290ac6848db05b7e2b227d4a85c24812..ca86ab05393ba73c0d20443f95ceb5e8c5023c4d 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp @@ -19,23 +19,39 @@ namespace DistributedDB { SQLiteSingleVerRelationalStorageExecutor::SQLiteSingleVerRelationalStorageExecutor(sqlite3 *dbHandle, bool writable) - : SQLiteStorageExecutor(dbHandle, writable, false) {}; + : SQLiteStorageExecutor(dbHandle, writable, false) +{} -int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, - const RelationalStoreDelegate::TableOption &option) +int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, TableInfo &table) { if (dbHandle_ == nullptr) { - LOGE("Begin transaction failed, dbHandle is null."); + LOGE("[CreateDistributedTable] Begin transaction failed, dbHandle is null."); return -E_INVALID_DB; } - TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); if (errCode != E_OK) { - LOGE("[CreateDistributedTable] analysis table schema failed"); + LOGE("[CreateDistributedTable] analysis table schema failed. %d", errCode); + return errCode; + } + + if (table.GetCreateTableSql().find("WITHOUT ROWID") != std::string::npos) { + LOGE("[CreateDistributedTable] Not support create distributed table without rowid."); + return -E_NOT_SUPPORT; + } + + bool isTableEmpty = false; + errCode = SQLiteUtils::CheckTableEmpty(dbHandle_, tableName, isTableEmpty); + if (errCode != E_OK) { + LOGE("[CreateDistributedTable] Check table [%s] is empty failed. %d", tableName.c_str(), errCode); return errCode; } + if (!isTableEmpty) { // create distributed table should on an empty table + LOGE("[CreateDistributedTable] Create distributed table should on an empty table."); + return -E_NOT_SUPPORT; + } + // create log table errCode = SQLiteUtils::CreateRelationalLogTable(dbHandle_, tableName); if (errCode != E_OK) { @@ -86,7 +102,7 @@ int SQLiteSingleVerRelationalStorageExecutor::Rollback() return errCode; } -int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) +int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(const QueryObject &query) { int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); if (errCode != E_OK) { @@ -95,141 +111,109 @@ int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) return errCode; } -// binding index just for the get sync data sql -static const int BIND_BEGIN_STAMP_INDEX = 1; -static const int BIND_END_STAMP_INDEX = 2; - -int SQLiteSingleVerRelationalStorageExecutor::PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, - sqlite3_stmt *&statement, bool getDeletedData) const -{ - if (dbHandle_ == nullptr) { - return -E_INVALID_DB; - } - - const std::string SELECT_SYNC_DELETED_ENTRIES_SQL = - "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + - "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x03) ORDER BY timestamp ASC;"; - const std::string SELECT_SYNC_ENTRIES_SQL = - "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + - "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x02=0x02) ORDER BY timestamp ASC;"; - - const std::string sql = (getDeletedData ? SELECT_SYNC_DELETED_ENTRIES_SQL : SELECT_SYNC_ENTRIES_SQL); - int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); - if (errCode != E_OK) { - LOGE("Prepare the sync entries statement error:%d", errCode); - return errCode; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_BEGIN_STAMP_INDEX, begin); - if (errCode != E_OK) { - goto ERROR; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_END_STAMP_INDEX, end); - if (errCode != E_OK) { - goto ERROR; - } - -ERROR: - if (errCode != E_OK) { - LOGE("Bind the timestamp for getting sync data error:%d", errCode); - SQLiteUtils::ResetStatement(statement, true, errCode); - } - - return CheckCorruptedStatus(errCode); -} - -static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) +static int GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) { - switch (type) { - case StorageType::STORAGE_TYPE_BOOL: { - value = static_cast(sqlite3_column_int64(statement, cid)); + int errCode = E_OK; + int storageType = sqlite3_column_type(statement, cid); + switch (storageType) { + case SQLITE_INTEGER: { + value = static_cast(sqlite3_column_int64(statement, cid)); break; } - - case StorageType::STORAGE_TYPE_INTEGER: { - int64_t v = static_cast(sqlite3_column_int64(statement, cid)); - value = v; + case SQLITE_FLOAT: { + value = sqlite3_column_double(statement, cid); break; } - - case StorageType::STORAGE_TYPE_REAL: { - value = sqlite3_column_double(statement, cid); + case SQLITE_BLOB: { + std::vector blobValue; + errCode = SQLiteUtils::GetColumnBlobValue(statement, cid, blobValue); + if (errCode != E_OK) { + return errCode; + } + auto blob = new (std::nothrow) Blob; + if (blob == nullptr) { + return -E_OUT_OF_MEMORY; + } + blob->WriteBlob(blobValue.data(), static_cast(blobValue.size())); + errCode = value.Set(blob); break; } - - case StorageType::STORAGE_TYPE_TEXT: { + case SQLITE_NULL: { + break; + } + case SQLITE3_TEXT: { const char *colValue = reinterpret_cast(sqlite3_column_text(statement, cid)); if (colValue == nullptr) { - value = std::string(); + value.ResetValue(); } else { value = std::string(colValue); + if (value.GetType() == StorageType::STORAGE_TYPE_NULL) { + errCode = -E_OUT_OF_MEMORY; + } } break; } - - case StorageType::STORAGE_TYPE_BLOB: { - std::vector blobValue; - (void)SQLiteUtils::GetColumnBlobValue(statement, cid, blobValue); - Blob blob; - blob.WriteBlob(blobValue.data(), static_cast(blobValue.size())); - value = blob; + default: { break; } - - default: - return; } + return errCode; } -static void BindDataValueByType(sqlite3_stmt *statement, - const std::optional &value, StorageType type, int cid) +static int BindDataValueByType(sqlite3_stmt *statement, const std::optional &data, int cid) { - if (value == std::nullopt) { - sqlite3_bind_null(statement, cid); - return; + if (!data.has_value()) { // For the column that added after enable distributed. + return E_OK; } + int errCode = E_OK; + StorageType type = data.value().GetType(); switch (type) { case StorageType::STORAGE_TYPE_BOOL: { bool boolData = false; - value.value().GetBool(boolData); - sqlite3_bind_int(statement, cid, boolData); + (void)data.value().GetBool(boolData); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_int(statement, cid, boolData)); break; } case StorageType::STORAGE_TYPE_INTEGER: { int64_t intData = 0; - value.value().GetInt64(intData); - sqlite3_bind_int64(statement, cid, intData); + (void)data.value().GetInt64(intData); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_int64(statement, cid, intData)); break; } case StorageType::STORAGE_TYPE_REAL: { double doubleData = 0; - value.value().GetDouble(doubleData); - sqlite3_bind_double(statement, cid, doubleData); + (void)data.value().GetDouble(doubleData); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_double(statement, cid, doubleData)); break; } case StorageType::STORAGE_TYPE_TEXT: { std::string strData; - value.value().GetText(strData); - SQLiteUtils::BindTextToStatement(statement, cid, strData); + (void)data.value().GetText(strData); + errCode = SQLiteUtils::BindTextToStatement(statement, cid, strData); break; } case StorageType::STORAGE_TYPE_BLOB: { Blob blob; - value.value().GetBlob(blob); + (void)data.value().GetBlob(blob); std::vector blobData(blob.GetData(), blob.GetData() + blob.GetSize()); - SQLiteUtils::BindBlobToStatement(statement, cid, blobData, true); + errCode = SQLiteUtils::BindBlobToStatement(statement, cid, blobData, true); + break; + } + + case StorageType::STORAGE_TYPE_NULL: { + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_null(statement, cid)); break; } default: - return; + break; } + return errCode; } static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) @@ -249,7 +233,6 @@ static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) return errCode; } logInfo.originDev = std::string(oriDev.begin(), oriDev.end()); - logInfo.timestamp = static_cast(sqlite3_column_int64(logStatement, 3)); logInfo.wTimeStamp = static_cast(sqlite3_column_int64(logStatement, 4)); logInfo.flag = static_cast(sqlite3_column_int64(logStatement, 5)); @@ -265,68 +248,6 @@ static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) return errCode; } -static sqlite3_stmt *GetDataStmtByPK(sqlite3 *db, const std::string &tableName, int rowId) -{ - std::string sql = "select * from " + tableName + " where rowId=?;"; - sqlite3_stmt *statement = nullptr; - int errCode = SQLiteUtils::GetStatement(db, sql, statement); - if (errCode != E_OK) { - LOGE("[data by rowid statement] Get statement fail!"); - return nullptr; - } - - errCode = SQLiteUtils::BindInt64ToStatement(statement, 1, rowId); - if (errCode != E_OK) { - SQLiteUtils::ResetStatement(statement, true, errCode); - return nullptr; - } - - return statement; -} - - -int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *logStatement, DataItem &dataItem) const -{ - std::map colInfos = table_.GetFields(); - RowDataWithLog data; - - int errCode = GetLogData(logStatement, data.logInfo); - if (errCode != E_OK) { - LOGE("relational data value transfer to kv fail"); - return errCode; - } - - // dataKey is rowid by pk - // only get local device data - sqlite3_stmt *stmt = GetDataStmtByPK(dbHandle_, table_.GetTableName(), data.logInfo.dataKey); - - errCode = SQLiteUtils::StepWithRetry(stmt, false); - if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW) && errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGE("[SQLiteSingleVerRelationalStorageExecutor] Step fail errCode:%d", errCode); - return errCode; - } - - std::vector fieldInfos; - for (const auto &col: colInfos) { - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - break; - } - DataValue value; - GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId()); - - fieldInfos.push_back(col.second); - data.rowData.push_back(value); - } - - SQLiteUtils::ResetStatement(stmt, true, errCode); - - errCode = DataTransformer::SerializeDataItem(data, fieldInfos, dataItem); - if (errCode != E_OK) { - LOGE("relational data value transfer to kv fail"); - } - return errCode; -} - static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) { // timestamp and local flag: 3 * uint64_t, version(uint32_t), key, value, origin dev and the padding size. @@ -338,57 +259,10 @@ static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) return dataSize; } -int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataItems(std::vector &dataItems, - sqlite3_stmt *logStatement, size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const -{ - int errCode; - size_t dataTotalSize = 0; - do { - DataItem item; - errCode = SQLiteUtils::StepWithRetry(logStatement, isMemDb_); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { - errCode = GetDataItemForSync(logStatement, item); - } else { - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); - errCode = -E_FINISHED; - } else { - LOGE("Get sync data error:%d", errCode); - } - break; - } - - // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. - dataTotalSize += GetDataItemSerialSize(item, appendLength); - if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || - dataItems.size() >= dataSizeInfo.packetSize) { - errCode = -E_UNFINISHED; - break; - } else { - dataItems.push_back(std::move(item)); - } - } while (true); - return errCode; -} - -int SQLiteSingleVerRelationalStorageExecutor::GetDeletedSyncDataByTimestamp(std::vector &dataItems, - size_t appendLength, TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const -{ - sqlite3_stmt *logStatement = nullptr; - int errCode = PrepareForSyncDataByTime(begin, end, logStatement, true); - if (errCode != E_OK) { - return errCode; - } - - errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); - SQLiteUtils::ResetStatement(logStatement, true, errCode); - return CheckCorruptedStatus(errCode); -} - -static const std::string SELECT_META_VALUE_SQL = - "SELECT value FROM distributeddatamgr_aux_metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &value) const { + static const std::string SELECT_META_VALUE_SQL = "SELECT value FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_META_VALUE_SQL, statement); if (errCode != E_OK) { @@ -414,9 +288,10 @@ int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &v return errCode; } -static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO distributeddatamgr_aux_metadata VALUES(?,?);"; int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const { + static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + + "metadata VALUES(?,?);"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, INSERT_META_SQL, statement); if (errCode != E_OK) { @@ -443,9 +318,10 @@ ERROR: return errCode; } -static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM distributeddatamgr_aux_metadata WHERE key=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const { + static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_SQL, statement); if (errCode != E_OK) { @@ -469,10 +345,10 @@ int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector=? AND key<=?;"; int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const { + static const std::string REMOVE_META_VALUE_BY_KEY_PREFIX_SQL = "DELETE FROM " + DBConstant::RELATIONAL_PREFIX + + "metadata WHERE key>=? AND key<=?;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_BY_KEY_PREFIX_SQL, statement); if (errCode != E_OK) { @@ -517,10 +393,9 @@ static int GetAllKeys(sqlite3_stmt *statement, std::vector &keys) return errCode; } - -static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM distributeddatamgr_aux_metadata;"; int SQLiteSingleVerRelationalStorageExecutor::GetAllMetaKeys(std::vector &keys) const { + static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM " + DBConstant::RELATIONAL_PREFIX + "metadata;"; sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_ALL_META_KEYS, statement); if (errCode != E_OK) { @@ -536,7 +411,7 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj const std::string &deviceName, sqlite3_stmt *&logStmt) const { std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_log"; + const std::string tableName = DBConstant::RELATIONAL_PREFIX + object.GetTableName() + "_log"; std::string dataFormat = "?, '" + deviceName + "', ?, ?, ?, ?, ?"; std::string sql = "INSERT OR REPLACE INTO " + tableName + @@ -552,10 +427,10 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObj int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const { - // distributeddatamgr_aux_userTableName_deviceHash + // naturalbase_rdb_aux_userTableName_deviceHash // tableName std::string devName = DBCommon::TransferHashString(deviceName); - const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_" + + const std::string tableName = DBConstant::RELATIONAL_PREFIX + object.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); TableInfo table; int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); @@ -587,13 +462,14 @@ int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryOb return errCode; } -int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statement, - const DataItem &dataItem, TimeStamp &maxTimestamp) +int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statement, const DataItem &dataItem, + TimeStamp &maxTimestamp) { Key hashKey; (void)DBCommon::CalcValueHash(dataItem.key, hashKey); std::string hash = std::string(hashKey.begin(), hashKey.end()); - std::string sql = "select * from distributeddatamgr_aux_" + table_.GetTableName() + "_log where hash_key = ?;"; + std::string sql = "select * from " + DBConstant::RELATIONAL_PREFIX + table_.GetTableName() + + "_log where hash_key = ?;"; sqlite3_stmt *queryStmt = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, queryStmt); if (errCode != E_OK) { @@ -651,7 +527,7 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statemen int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem) { std::string devName = DBCommon::TransferHashString(dataItem.dev); - const std::string tableName = "distributeddatamgr_aux_" + table_.GetTableName() + "_" + + const std::string tableName = DBConstant::RELATIONAL_PREFIX + table_.GetTableName() + "_" + DBCommon::TransferStringToHex(devName); std::string hashKey = std::string(dataItem.hashKey.begin(), dataItem.hashKey.end()); std::string sql = "DELETE FROM " + tableName + " WHERE calc_hash(" + table_.GetPrimaryKey() + ")=" + hashKey + ";"; @@ -694,12 +570,10 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncDataItem(sqlite3_stmt *sta for (size_t index = 0; index < data.optionalData.size(); index++) { const auto &filedData = data.optionalData[index]; - if (filedData.has_value()) { - (void)BindDataValueByType(statement, filedData, - filedData.value().GetType(), fieldInfos[index].GetColumnId() + 1); - } else { - (void)BindDataValueByType(statement, filedData, - StorageType::STORAGE_TYPE_NULL, fieldInfos[index].GetColumnId() + 1); + errCode = BindDataValueByType(statement, filedData, fieldInfos[index].GetColumnId() + 1); + if (errCode != E_OK) { + LOGE("Bind data failed, errCode:%d, cid:%d.", errCode, fieldInfos[index].GetColumnId() + 1); + return errCode; } } @@ -771,41 +645,87 @@ int SQLiteSingleVerRelationalStorageExecutor::SaveSyncItems(const QueryObject &o return errCode; } -static int GetLogInfoStatement(sqlite3 *dbHandle, const std::string &tableName, - uint64_t beginTime, uint64_t endTime, sqlite3_stmt *&statement) +int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *stmt, DataItem &dataItem, + bool isGettingDeletedData) const { - std::string sql = "select * from distributeddatamgr_aux_" + tableName + - "_log where flag=0x02 AND timestamp>=? AND timestamp fieldInfos; + if (!isGettingDeletedData) { + for (const auto &col: table_.GetFields()) { + LOGD("[GetDataItemForSync] field:%s type:%d cid:%d", col.second.GetFieldName().c_str(), + col.second.GetStorageType(), col.second.GetColumnId() + 7); + DataValue value; + errCode = GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId() + 7); + if (errCode != E_OK) { + return errCode; + } + fieldInfos.push_back(col.second); + data.rowData.push_back(value); + } } - errCode = SQLiteUtils::BindInt64ToStatement(statement, 2, endTime); + errCode = DataTransformer::SerializeDataItem(data, fieldInfos, dataItem); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(statement, true, errCode); - return errCode; + LOGE("relational data value transfer to kv fail"); } return errCode; } int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, - QueryObject query, const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const + const DataSizeSpecInfo &dataSizeInfo, std::function getStmt) { - sqlite3_stmt *logStatement = nullptr; - int errCode = GetLogInfoStatement(dbHandle_, query.GetTableName(), timeRange.first, timeRange.second, logStatement); - if (errCode == E_OK) { - errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); + sqlite3_stmt *stmt = nullptr; + bool isGettingDeletedData = false; + int errCode = getStmt(dbHandle_, stmt, isGettingDeletedData); + if (errCode != E_OK) { + return errCode; } - SQLiteUtils::ResetStatement(logStatement, true, errCode); + + size_t dataTotalSize = 0; + do { + DataItem item; + errCode = SQLiteUtils::StepWithRetry(stmt, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + errCode = GetDataItemForSync(stmt, item, isGettingDeletedData); + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); + errCode = -E_FINISHED; + break; + } else { + LOGE("Get sync data error:%d", errCode); + break; + } + + // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. + dataTotalSize += GetDataItemSerialSize(item, appendLength); + if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || + dataItems.size() >= dataSizeInfo.packetSize) { + errCode = -E_UNFINISHED; + break; + } else { + dataItems.push_back(std::move(item)); + } + } while (true); + + SQLiteUtils::ResetStatement(stmt, true, errCode); return errCode; } + +int SQLiteSingleVerRelationalStorageExecutor::CheckDBModeForRelational() +{ + std::string journalMode; + int errCode = SQLiteUtils::GetJournalMode(dbHandle_, journalMode); + if (errCode != E_OK || journalMode != "wal") { + LOGE("Not support journal mode %s for relational db, expect wal mode, %d", journalMode.c_str(), errCode); + return -E_NOT_SUPPORT; + } + return E_OK; +} } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h index 301204f61a909cb08b0af0d94146015c7b0295df..ba0057ae17186556081e6f0c12097512a8692b0b 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -16,8 +16,9 @@ #define SQLITE_SINGLE_VER_RELATIONAL_STORAGE_EXECUTOR_H #ifdef RELATIONAL_STORE -#include "macro_utils.h" +#include "data_transformer.h" #include "db_types.h" +#include "macro_utils.h" #include "sqlite_utils.h" #include "sqlite_storage_executor.h" #include "relational_store_delegate.h" @@ -32,19 +33,17 @@ public: // Delete the copy and assign constructors DISABLE_COPY_ASSIGN_MOVE(SQLiteSingleVerRelationalStorageExecutor); - int CreateDistributedTable(const std::string &tableName, const RelationalStoreDelegate::TableOption &option); + int CreateDistributedTable(const std::string &tableName, TableInfo &table); int StartTransaction(TransactType type); int Commit(); int Rollback(); - int SetTableInfo(QueryObject query); + int SetTableInfo(const QueryObject &query); // For Get sync data - int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, QueryObject query, - const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const; - int GetDeletedSyncDataByTimestamp(std::vector &dataItems, size_t appendLength, - TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const; + int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, + const DataSizeSpecInfo &dataSizeInfo, std::function getStmt); // operation of meta data int GetKvData(const Key &key, Value &value) const; @@ -57,14 +56,15 @@ public: int SaveSyncItems(const QueryObject &object, std::vector &dataItems, const std::string &deviceName, TimeStamp &timeStamp); + int AnalysisRelationalSchema(const std::string &tableName, TableInfo &tableInfo); + + int CheckDBModeForRelational(); + private: int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData) const; - int GetSyncDataItems(std::vector &dataItems, sqlite3_stmt *statement, - size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const; - - int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem) const; + int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem, bool isGettingDeletedData) const; int SaveSyncDataItems(const QueryObject &object, std::vector &dataItems, const std::string &deviceName, TimeStamp &timeStamp); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp index b99b34f90d97a77d2764ebe1e555ee647ce7993d..8ae53a1ac9181c2e3942306f00dea7ea7f541880 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp @@ -206,7 +206,7 @@ int SQLiteSingleVerStorageEngine::AddSubscribeToMainDBInMigrate() int errCode = E_OK; auto handle = static_cast(FindExecutor(true, OperatePerm::NORMAL_PERM, errCode)); if (errCode != E_OK || handle == nullptr) { - LOGE("Get available executor for add subscribe failed. %s", errCode); + LOGE("Get available executor for add subscribe failed. %d", errCode); return errCode; } errCode = handle->StartTransaction(TransactType::IMMEDIATE); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index 9d3e9a2ff5d71d252f84c0a571eee3595046924b..84d5b2348382b149fa986eb23356fcaa0b1caf3e 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -1298,9 +1298,9 @@ int SQLiteSingleVerStorageExecutor::PrepareForNotifyConflictAndObserver(DataItem LOGD("[SingleVerExe] Ignore the sync data."); if (isSyncMigrating_) { ResetForMigrateCacheData(); - return -E_IGNOR_DATA; + return -E_IGNORE_DATA; } - return ResetSaveSyncStatements(-E_IGNOR_DATA); + return ResetSaveSyncStatements(-E_IGNORE_DATA); } notify.dataStatus = JudgeSyncSaveType(dataItem, notify.getData, deviceInfo.deviceName, isHashKeyExisted); @@ -1343,7 +1343,7 @@ int SQLiteSingleVerStorageExecutor::SaveSyncDataItem(DataItem &dataItem, const D int errCode = PrepareForNotifyConflictAndObserver(dataItem, deviceInfo, notify); if (errCode != E_OK) { - if (errCode == -E_IGNOR_DATA) { + if (errCode == -E_IGNORE_DATA) { errCode = E_OK; } return errCode; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp index 872e873613f4d6e68993aaf08d497bee2732be84..9559b8fb4e28c65381fe07f35639af400639dc33 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp @@ -760,7 +760,7 @@ int SQLiteSingleVerStorageExecutor::PutIntoConflictAndCommitForMigrateCache(Data int errCode = PrepareForNotifyConflictAndObserver(dataItem, deviceInfo, notify); if (errCode != E_OK) { errCode = (errCode == -E_NOT_FOUND ? E_OK : errCode); - if (errCode == -E_IGNOR_DATA) { + if (errCode == -E_IGNORE_DATA) { notify.dataStatus.isDefeated = true; errCode = E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index 999d6f49fb57a40bd54ef70a8311f8c13cb75dbe..88cabd62ec53b5aaf5038a466a120ed6b4349465 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -102,7 +102,7 @@ std::string GetTriggerModeString(TriggerModeEnum mode) } } -int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp) +int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp, bool setWal) { uint64_t flag = SQLITE_OPEN_URI | SQLITE_OPEN_READWRITE; if (properties.createIfNecessary) { @@ -114,7 +114,10 @@ int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&db return -E_INVALID_ARGS; } std::string defaultAttachCipher = DEFAULT_ATTACH_CIPHER + cipherName + ";"; - std::vector sqls {WAL_MODE_SQL, defaultAttachCipher, DEFAULT_ATTACH_KDF_ITER}; + std::vector sqls {defaultAttachCipher, DEFAULT_ATTACH_KDF_ITER}; + if (setWal) { + sqls.push_back(WAL_MODE_SQL); + } std::string fileUrl = DBConstant::SQLITE_URL_PRE + properties.uri; int errCode = sqlite3_open_v2(fileUrl.c_str(), &dbTemp, flag, nullptr); @@ -139,7 +142,7 @@ END: return errCode; } -int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db) +int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db, bool setWal) { { // Only for register the sqlite3 log callback @@ -150,7 +153,7 @@ int SQLiteUtils::OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db) } } sqlite3 *dbTemp = nullptr; - int errCode = CreateDataBase(properties, dbTemp); + int errCode = CreateDataBase(properties, dbTemp, setWal); if (errCode != E_OK) { goto END; } @@ -902,6 +905,56 @@ int SQLiteUtils::GetVersion(sqlite3 *db, int &version) return errCode; } +int SQLiteUtils::GetJournalMode(sqlite3 *db, std::string &mode) +{ + if (db == nullptr) { + return -E_INVALID_DB; + } + + std::string sql = "PRAGMA journal_mode;"; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_prepare(db, sql.c_str(), -1, &statement, nullptr); + if (errCode != SQLITE_OK || statement == nullptr) { + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + return errCode; + } + + if (sqlite3_step(statement) == SQLITE_ROW) { + errCode = SQLiteUtils::GetColumnTextValue(statement, 0, mode); + } else { + LOGE("[SqlUtil][GetJournal] Get db journal_mode failed."); + errCode = SQLiteUtils::MapSQLiteErrno(SQLITE_ERROR); + } + + SQLiteUtils::ResetStatement(statement, true, errCode); + return E_OK; +} + +int SQLiteUtils::GetSynchronousMode(sqlite3 *db, int &mode) +{ + if (db == nullptr) { + return -E_INVALID_DB; + } + + std::string sql = "PRAGMA synchronous;"; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_prepare(db, sql.c_str(), -1, &statement, nullptr); + if (errCode != SQLITE_OK || statement == nullptr) { + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + return errCode; + } + + if (sqlite3_step(statement) == SQLITE_ROW) { + mode = sqlite3_column_int(statement, 0); + } else { + LOGE("[SqlUtil][GetSynchronous] Get db synchronous failed."); + errCode = SQLiteUtils::MapSQLiteErrno(SQLITE_ERROR); + } + + SQLiteUtils::ResetStatement(statement, true, errCode); + return E_OK; +} + int SQLiteUtils::SetUserVer(const OpenDbProperties &properties, int version) { if (properties.uri.empty()) { @@ -1252,7 +1305,7 @@ int SQLiteUtils::RegisterGetSysTime(sqlite3 *db) int SQLiteUtils::CreateRelationalMetaTable(sqlite3 *db) { std::string sql = - "CREATE TABLE IF NOT EXISTS distributeddatamgr_aux_metadata(" \ + "CREATE TABLE IF NOT EXISTS " + DBConstant::RELATIONAL_PREFIX + "metadata(" \ "key BLOB PRIMARY KEY NOT NULL," \ "value BLOB);"; @@ -1267,21 +1320,21 @@ int SQLiteUtils::CreateRelationalMetaTable(sqlite3 *db) int SQLiteUtils::CreateRelationalLogTable(sqlite3 *db, const std::string &oriTableName) { std::string sql = - "CREATE TABLE IF NOT EXISTS distributeddatamgr_aux_" + oriTableName + "_log(" \ + "CREATE TABLE IF NOT EXISTS " + DBConstant::RELATIONAL_PREFIX + oriTableName + "_log(" \ "data_key INT NOT NULL," \ "device BLOB," \ "ori_device BLOB," \ "timestamp INT NOT NULL," \ "wtimestamp INT NOT NULL," \ "flag INT NOT NULL," \ - "hash_key BLOB PRIMARY KEY NOT NULL);"; + "hash_key BLOB NOT NULL," + "PRIMARY KEY(device,hash_key));"; int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); if (errCode != E_OK) { LOGE("[SQLite] execute create table sql failed"); - return errCode; } - return E_OK; + return errCode; } template @@ -1296,10 +1349,10 @@ static std::string string_format(const std::string& format, Args ... args) int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &table) { std::string insertTrigger = - "CREATE TRIGGER IF NOT EXISTS distributeddatamgr_%s_ON_INSERT AFTER INSERT \n" \ + "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_INSERT AFTER INSERT \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t INSERT OR REPLACE INTO distributeddatamgr_aux_%s_log \ + "\t INSERT OR REPLACE INTO " + DBConstant::RELATIONAL_PREFIX + "%s_log \ (data_key, device, ori_device, timestamp, wtimestamp, flag, hash_key)" \ "VALUES (new.rowid, '%s', '%s', get_sys_time(), get_sys_time(), 0x02, calc_hash(new.%s));\n" \ "END;"; @@ -1307,19 +1360,19 @@ int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &tabl table.GetTableName().c_str(), table.GetDevId().c_str(), table.GetDevId().c_str(), table.GetPrimaryKey().c_str()); std::string updateTrigger = - "CREATE TRIGGER IF NOT EXISTS distributeddatamgr_%s_ON_UPDATE AFTER UPDATE \n" \ + "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_UPDATE AFTER UPDATE \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t UPDATE distributeddatamgr_aux_%s_log SET timestamp=get_sys_time(), device='%s' \ + "\t UPDATE " + DBConstant::RELATIONAL_PREFIX + "%s_log SET timestamp=get_sys_time(), device='%s' \ where hash_key=calc_hash(old.%s) and flag&0x10=0;\n" \ "END;"; updateTrigger = string_format(updateTrigger, table.GetTableName().c_str(), table.GetTableName().c_str(), table.GetTableName().c_str(), table.GetDevId().c_str(), table.GetPrimaryKey().c_str()); std::string deleteTrigger = - "CREATE TRIGGER IF NOT EXISTS distributeddatamgr_%s_ON_DELETE BEFORE DELETE \n" \ + "CREATE TRIGGER IF NOT EXISTS naturalbase_rdb_%s_ON_DELETE BEFORE DELETE \n" \ "ON %s\n" \ "BEGIN\n" \ - "\t UPDATE distributeddatamgr_aux_%s_log set flag=0x03,timestamp=get_sys_time() \ + "\t UPDATE " + DBConstant::RELATIONAL_PREFIX + "%s_log set flag=0x03,timestamp=get_sys_time() \ WHERE hash_key=calc_hash(old.%s);\n" \ "END;"; deleteTrigger = string_format(deleteTrigger, table.GetTableName().c_str(), @@ -1350,9 +1403,8 @@ int SQLiteUtils::CreateSameStuTable(sqlite3 *db, const std::string &oriTableName int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); if (errCode != E_OK) { LOGE("[SQLite] execute create table sql failed"); - return errCode; } - return E_OK; + return errCode; } int SQLiteUtils::RegisterFunction(sqlite3 *db, const std::string &funcName, int nArg, void *uData, TransactFunc &func) @@ -1921,4 +1973,31 @@ int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) sqlite3_free(eSql); return E_OK; } + +int SQLiteUtils::CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool &isEmpty) +{ + if (db == nullptr) { + return -E_INVALID_ARGS; + } + + std::string cntSql = "SELECT min(rowid) FROM " + tableName + ";"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, cntSql, stmt); + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + if (sqlite3_column_type(stmt, 0) == SQLITE_NULL) { + isEmpty = true; + } else { + isEmpty = false; + } + errCode = E_OK; + } + + SQLiteUtils::ResetStatement(stmt, true, errCode); + return SQLiteUtils::MapSQLiteErrno(errCode); +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 94d7140010cec92e15b9bc74a1a5319ee507b2c4..8a53c5e12aed419ae484d1342d774c6881e6056f 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -21,7 +21,6 @@ #include "sqlite_import.h" #include "db_types.h" -#include "iprocess_system_api_adapter.h" #include "schema_object.h" #include "types.h" #ifdef RELATIONAL_STORE @@ -53,15 +52,15 @@ std::string GetTriggerModeString(TriggerModeEnum mode); } struct OpenDbProperties { - std::string uri{}; + std::string uri {}; bool createIfNecessary = true; bool isMemDb = false; - std::vector sqls{}; + std::vector sqls {}; CipherType cipherType = CipherType::AES_256_GCM; - CipherPassword passwd{}; - std::string schema{}; - std::string subdir{}; - SecurityOption securityOpt{}; + CipherPassword passwd {}; + std::string schema {}; + std::string subdir {}; + SecurityOption securityOpt {}; int conflictReslovePolicy = DEFAULT_LAST_WIN; bool createDirByStoreIdOnly = false; }; @@ -69,7 +68,7 @@ struct OpenDbProperties { class SQLiteUtils { public: // Initialize the SQLiteUtils with the given properties. - static int OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db); + static int OpenDatabase(const OpenDbProperties &properties, sqlite3 *&db, bool setWal = true); // Check the statement and prepare the new if statement is null static int GetStatement(sqlite3 *db, const std::string &sql, sqlite3_stmt *&statement); @@ -117,6 +116,10 @@ public: static int GetVersion(sqlite3 *db, int &version); + static int GetJournalMode(sqlite3 *db, std::string &mode); + + static int GetSynchronousMode(sqlite3 *db, int &mode); + static int SetUserVer(const OpenDbProperties &properties, int version); static int SetUserVer(sqlite3 *db, int version); @@ -174,9 +177,11 @@ public: static int ExpandedSql(sqlite3_stmt *stmt, std::string &basicString); + static int CheckTableEmpty(sqlite3 *db, const std::string &tableName, bool &isEmpty); + private: - static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp); + static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp, bool setWal); static int SetBusyTimeout(sqlite3 *db, int timeout); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp index 5ae456f8f4c701ca2f362a24ac4455d0e5c90a0e..f2ca51301c0e5a136aaecdb9e50fbc65377aa69f 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -26,8 +26,7 @@ SyncAbleEngine::SyncAbleEngine(ISyncInterface *store) {} SyncAbleEngine::~SyncAbleEngine() -{ -} +{} // Start a sync action. int SyncAbleEngine::Sync(const ISyncer::SyncParma &parm) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3086ba4d3cffa5a6464e521f06a2fee4cb548094 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.cpp @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "log_print.h" +#include "version.h" +#include "single_ver_data_sync.h" +#include "single_ver_data_message_schedule.h" + +namespace DistributedDB { +SingleVerDataMessageSchedule::~SingleVerDataMessageSchedule() +{ + LOGD("~SingleVerDataMessageSchedule"); + ClearMsg(); +} + +void SingleVerDataMessageSchedule::Initialize(const std::string &label, const std::string &deviceId) +{ + label_ = label; + deviceId_ = deviceId; +} + +void SingleVerDataMessageSchedule::PutMsg(Message *inMsg) +{ + if (inMsg == nullptr) { + return; + } + std::lock_guard lock(queueLock_); + msgQueue_.push(inMsg); + isNeedReload_ = true; +} + +bool SingleVerDataMessageSchedule::IsNeedReloadQueue() +{ + std::lock_guard lock(queueLock_); + return isNeedReload_; +} + +Message *SingleVerDataMessageSchedule::MoveNextMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, + bool &isNeedContinue) +{ + uint32_t remoteVersion = context->GetRemoteSoftwareVersion(); + if (remoteVersion < SOFTWARE_VERSION_RELEASE_3_0) { + // just get last msg when remote version is < 103 or >=103 but just open db now + return GetLastMsgFromQueue(); + } + { + std::lock_guard lock(workingLock_); + if (isWorking_) { + isNeedContinue = false; + return nullptr; + } + isWorking_ = true; + } + ResetTimer(context); + UpdateMsgMap(); + Message *msg = GetMsgFromMap(isNeedHandle); + isNeedContinue = true; + if (msg == nullptr) { + std::lock_guard lock(workingLock_); + isWorking_ = false; + return nullptr; + } + return msg; +} + +void SingleVerDataMessageSchedule::ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, + const Message *inMsg) +{ + if (isNeedHandleStatus) { + const DataRequestPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + LOGE("[DataMsgSchedule] packet is nullptr"); + return; + } + uint64_t curPacketId = packet->GetPacketId(); + { + std::lock_guard lock(lock_); + finishedPacketId_ = curPacketId; + if (isNeedClearMap) { + ClearMsgMapWithNoLock(); + expectedSequenceId_ = 1; + } else { + LOGI("[DataMsgSchedule] DealMsg seqId=%u finishedPacketId=%llu ok,label=%s,deviceId=%s{private}", + expectedSequenceId_, finishedPacketId_, label_.c_str(), deviceId_.c_str()); + expectedSequenceId_++; + } + } + } + std::lock_guard lock(workingLock_); + isWorking_ = false; +} + +void SingleVerDataMessageSchedule::ClearMsg() +{ + StopTimer(); + ClearMsgQueue(); + ClearMsgMap(); +} + +void SingleVerDataMessageSchedule::UpdateMsgMap() +{ + std::queue msgTmpQueue; + { + std::lock_guard lock(queueLock_); + while (!msgQueue_.empty()) { + msgTmpQueue.push(msgQueue_.front()); + msgQueue_.pop(); + } + isNeedReload_ = false; + } + UpdateMsgMapInner(msgTmpQueue); +} + +void SingleVerDataMessageSchedule::UpdateMsgMapInner(std::queue &msgTmpQueue) +{ + // update msg map + std::lock_guard lock(lock_); + while (!msgTmpQueue.empty()) { + Message *msg = msgTmpQueue.front(); + msgTmpQueue.pop(); + if (msg == nullptr) { + continue; + } + const DataRequestPacket *packet = msg->GetObject(); + if (packet == nullptr) { + delete msg; + continue; + } + uint32_t sessionId = msg->GetSessionId(); + uint32_t sequenceId = msg->GetSequenceId(); + uint64_t packetId = packet->GetPacketId(); + if (prevSessionId_ != 0 && sessionId == prevSessionId_) { + LOGD("[DataMsgSchedule] recv prev sessionId msg,drop msg,label=%s,deviceId=%s{private}", + label_.c_str(), deviceId_.c_str()); + delete msg; + continue; + } + if (sessionId != currentSessionId_) { + // make sure all msg sessionId is same in msgMap + ClearMsgMapWithNoLock(); + prevSessionId_ = currentSessionId_; + currentSessionId_ = sessionId; + finishedPacketId_ = 0; + expectedSequenceId_ = 1; + } + if (messageMap_.count(sequenceId) > 0) { + const auto *cachePacket = messageMap_[sequenceId]->GetObject(); + LOGD("[DataMsgSchedule] msg packetId=%llu,cachePacketId=%llu,label=%s,deviceId=%s", packetId, + cachePacket->GetPacketId(), label_.c_str(), deviceId_.c_str()); + if (packetId != 0 && packetId < cachePacket->GetPacketId()) { + delete msg; + continue; + } + delete messageMap_[sequenceId]; + messageMap_[sequenceId] = nullptr; + } + messageMap_[sequenceId] = msg; + LOGD("[DataMsgSchedule] put into msgMap seqId=%llu,packetId=%llu,label=%s,deviceId=%s", sequenceId, + packetId, label_.c_str(), deviceId_.c_str()); + } +} + +Message *SingleVerDataMessageSchedule::GetMsgFromMap(bool &isNeedHandle) +{ + isNeedHandle = true; + std::lock_guard lock(lock_); + while (!messageMap_.empty()) { + auto iter = messageMap_.begin(); + Message *msg = iter->second; + messageMap_.erase(iter); + const DataRequestPacket *packet = msg->GetObject(); + if (packet == nullptr) { + LOGE("[DataMsgSchedule] expected error"); + delete msg; + continue; + } + uint32_t sequenceId = msg->GetSequenceId(); + uint64_t packetId = packet->GetPacketId(); + if (sequenceId < expectedSequenceId_) { + uint64_t revisePacketId = finishedPacketId_ - (expectedSequenceId_ - 1 - sequenceId); + LOGI("[DataMsgSchedule] msg seqId=%llu less than exSeqId=%llu,pacId=%llu,revisePacId=%llu,label=%s,dev=%s", + sequenceId, expectedSequenceId_, packetId, revisePacketId, label_.c_str(), deviceId_.c_str()); + if (packetId < revisePacketId) { + delete msg; + continue; + } + // means already handle the msg, and just send E_OK ack in dataSync + isNeedHandle = false; + return msg; + } + if (sequenceId == expectedSequenceId_) { + if (packetId < finishedPacketId_) { + LOGI("[DataMsgSchedule] drop msg seqId=%llu,packetId=%llu,label=%s,deviceId=%s", sequenceId, + packetId, label_.c_str(), deviceId_.c_str()); + delete msg; + continue; + } + // if packetId == finishedPacketId_ need handle + // it will happened while watermark/need_abilitySync when last ack is missing + return msg; + } + // sequenceId > expectedSequenceId_, not need handle, put into map again + messageMap_[sequenceId] = msg; + return nullptr; + } + return nullptr; +} + +Message *SingleVerDataMessageSchedule::GetLastMsgFromQueue() +{ + std::lock_guard lock(queueLock_); + isNeedReload_ = false; + while (!msgQueue_.empty()) { + Message *msg = msgQueue_.front(); + msgQueue_.pop(); + if (msgQueue_.empty()) { // means last msg + return msg; + } + delete msg; + } + return nullptr; +} + +void SingleVerDataMessageSchedule::ClearMsgMap() +{ + std::lock_guard lock(lock_); + ClearMsgMapWithNoLock(); +} + +void SingleVerDataMessageSchedule::ClearMsgMapWithNoLock() +{ + LOGD("[DataMsgSchedule] begin to ClearMsgMapWithNoLock"); + for (auto &iter : messageMap_) { + delete iter.second; + iter.second = nullptr; + } + messageMap_.clear(); +} + +void SingleVerDataMessageSchedule::ClearMsgQueue() +{ + std::lock_guard lock(queueLock_); + while (!msgQueue_.empty()) { + Message *msg = msgQueue_.front(); + msgQueue_.pop(); + delete msg; + } +} + +void SingleVerDataMessageSchedule::StartTimer(SingleVerSyncTaskContext *context) +{ + std::lock_guard lock(lock_); + TimerId timerId = 0; + RefObject::IncObjRef(context); + TimerAction timeOutCallback = std::bind(&SingleVerDataMessageSchedule::TimeOut, this, std::placeholders::_1); + int errCode = RuntimeContext::GetInstance()->SetTimer(IDLE_TIME_OUT, timeOutCallback, + [context]() { + int errCode = RuntimeContext::GetInstance()->ScheduleTask([context]() { + RefObject::DecObjRef(context); + }); + if (errCode != E_OK) { + LOGE("[DataMsgSchedule] timer finalizer ScheduleTask,errCode=%d", errCode); + } + }, timerId); + if (errCode != E_OK) { + RefObject::DecObjRef(context); + LOGE("[DataMsgSchedule] timer ScheduleTask, errCode=%d", errCode); + return; + } + timerId_ = timerId; + LOGD("[DataMsgSchedule] StartTimer,TimerId=%llu", timerId_); +} + +void SingleVerDataMessageSchedule::StopTimer() +{ + TimerId timerId; + { + std::lock_guard lock(lock_); + LOGD("[DataMsgSchedule] StopTimer,remove TimerId=%llu", timerId_); + if (timerId_ == 0) { + return; + } + timerId = timerId_; + timerId_ = 0; + } + RuntimeContext::GetInstance()->RemoveTimer(timerId); +} + +void SingleVerDataMessageSchedule::ResetTimer(SingleVerSyncTaskContext *context) +{ + StopTimer(); + StartTimer(context); +} + +int SingleVerDataMessageSchedule::TimeOut(TimerId timerId) +{ + if (IsNeedReloadQueue()) { + LOGI("[DataMsgSchedule] new msg exists, no need to timeout handle"); + return E_OK; + } + { + std::lock_guard lock(workingLock_); + if (isWorking_) { + LOGI("[DataMsgSchedule] other thread is handle msg, no need to timeout handle"); + return E_OK; + } + } + { + std::lock_guard lock(lock_); + LOGI("[DataMsgSchedule] timeout handling, stop timerId_[%llu]", timerId, timerId_); + if (timerId == timerId_) { + ClearMsgMapWithNoLock(); + timerId_ = 0; + } + } + RuntimeContext::GetInstance()->RemoveTimer(timerId); + return E_OK; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h old mode 100755 new mode 100644 similarity index 33% rename from services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h index 461ac6465ce5a26cb3e30a7b62cdb0b6c1c71a83..9189a7a88290436d01aadef455b96c80b7bcc5a7 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_message_schedule.h @@ -13,57 +13,62 @@ * limitations under the License. */ -#ifndef SLIDING_WINDOW_RECEIVER_H -#define SLIDING_WINDOW_RECEIVER_H - +#ifndef SINGLE_VER_DATA_MESSAGE_SCHEDULE_H +#define SINGLE_VER_DATA_MESSAGE_SCHEDULE_H #include +#include +#include +#include #include "message.h" -#include "single_ver_data_sync.h" +#include "runtime_context.h" #include "single_ver_sync_task_context.h" namespace DistributedDB { -class SlidingWindowReceiver { +class SingleVerDataMessageSchedule { public: - SlidingWindowReceiver() = default; - ~SlidingWindowReceiver(); - DISABLE_COPY_ASSIGN_MOVE(SlidingWindowReceiver); - - int Initialize(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int Receive(Message *inMsg); - void Clear(); + SingleVerDataMessageSchedule() = default; + ~SingleVerDataMessageSchedule(); + void Initialize(const std::string &label, const std::string &deviceId); + void PutMsg(Message *inMsg); + bool IsNeedReloadQueue(); + Message *MoveNextMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, bool &isNeedContinue); + void ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, const Message *inMsg); + void ClearMsg(); private: - int PutMsg(Message *inMsg); - void DealMsg(); - int TimeOut(TimerId timerId); - void StartTimer(); + void UpdateMsgMap(); + void UpdateMsgMapInner(std::queue &msgTmpQueue); + Message *GetMsgFromMap(bool &isNeedHandle); + Message *GetLastMsgFromQueue(); + void ClearMsgMap(); + void ClearMsgMapWithNoLock(); + void ClearMsgQueue(); + void StartTimer(SingleVerSyncTaskContext *context); void StopTimer(); - static void StopTimer(TimerId timerId); - void ClearMap(); - void ResetInfo(); - int ErrHandle(uint32_t sequenceId, uint64_t packetId); - void SetEndField(bool isLastSequence, uint32_t sequenceId); + void ResetTimer(SingleVerSyncTaskContext *context); + // when timeout queue size is 0 because thread can move queue msg to map if isNeedReload which is + // actived when queue has new msg is true + // so only need clear map msg + int TimeOut(TimerId timerId); + static constexpr int IDLE_TIME_OUT = 5 * 60 * 1000; // 5min + std::mutex queueLock_; + std::queue msgQueue_; + bool isNeedReload_ = false; + // only one thread is deal msg + std::mutex workingLock_; + bool isWorking_ = false; + // first:sequenceId second:Message*, deal msg from low sequenceId to high sequenceId std::mutex lock_; - std::condition_variable workingTaskcv_; - // 0 is default invalid. - uint32_t sessionId_ = 0; - // first:sequenceId second:Message*, message data not deal. std::map messageMap_; - // 0 is has finished nothing; e.g. 3 is sequenceId 1 2 3 has finished, sequenceId 4 has not finished. - uint32_t hasFinishedMaxId_ = 0; - uint64_t hasFinishedPacketId_ = 0; - // 0 is idle - uint32_t workingId_ = 0; - // 0 is has not received the end packet now. - uint32_t endId_ = 0; - bool isWaterMarkErrHappened_ = false; - // timeout for idle wait packet + uint32_t prevSessionId_ = 0; // drop the msg if msg sessionId is prev sessionId. + uint32_t currentSessionId_ = 0; + uint64_t finishedPacketId_ = 0; // next msg packetId should larger than it + uint32_t expectedSequenceId_ = 0; // handle next msg which sequenceId is equal to it TimerId timerId_ = 0; - static constexpr int IDLE_TIME_OUT = 5 * 60 * 1000; // 5min - SingleVerSyncTaskContext *context_ = nullptr; - std::shared_ptr dataSync_; -}; -} // namespace DistributedDB -#endif // SLIDING_WINDOW_RECEIVER_H + std::string label_; + std::string deviceId_; +}; +} +#endif // SINGLE_VER_DATA_MESSAGE_SCHEDULE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index 46cd9692e0d2a77f98dffb80477adae70e136b59..23f669705bc568cbbce8b46f3970f6ca6ae3b4af 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -74,9 +74,152 @@ int SingleVerDataSync::Initialize(ISyncInterface *inStorage, ICommunicator *inCo label.resize(3); // only show 3 Bytes enough label_ = DBCommon::VectorToHexString(label); deviceId_ = deviceId; + msgSchedule_.Initialize(label_, deviceId_); return E_OK; } +int SingleVerDataSync::SyncStart(int mode, SingleVerSyncTaskContext *context) +{ + std::lock_guard lock(lock_); + if (sessionId_ != 0) { // auto sync timeout resend + return ReSendData(context); + } + ResetSyncStatus(mode, context); + LOGI("[DataSync] SendStart,mode=%d,label=%s,device=%s", mode_, label_.c_str(), STR_MASK(deviceId_)); + int tmpMode = SyncOperation::TransferSyncMode(mode); + int errCode = E_OK; + if (tmpMode == SyncModeType::PUSH) { + errCode = PushStart(context); + } else if (tmpMode == SyncModeType::PUSH_AND_PULL) { + errCode = PushPullStart(context); + } else if (tmpMode == SyncModeType::PULL) { + errCode = PullRequestStart(context); + } else { + errCode = PullResponseStart(context); + } + if (context->IsSkipTimeoutError(errCode)) { + // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. + // just return to avoid higher pressure for send. + return E_OK; + } + if (errCode != E_OK) { + LOGE("[DataSync] SendStart errCode=%d", errCode); + return errCode; + } + if (tmpMode == SyncModeType::PUSH_AND_PULL && context->GetTaskErrCode() == -E_EKEYREVOKED) { + LOGE("wait for recv finished for push and pull mode"); + return -E_EKEYREVOKED; + } + return InnerSyncStart(context); +} + +int SingleVerDataSync::InnerSyncStart(SingleVerSyncTaskContext *context) +{ + while (true) { + if (windowSize_ <= 0 || isAllDataHasSent_) { + LOGD("[DataSync] InnerDataSync winSize=%d,isAllSent=%d,label=%s,device=%s", windowSize_, isAllDataHasSent_, + label_.c_str(), STR_MASK(deviceId_)); + return E_OK; + } + int mode = SyncOperation::TransferSyncMode(mode_); + if (mode == SyncModeType::PULL) { + LOGE("[DataSync] unexpected error"); + return -E_INVALID_ARGS; + } + int errCode; + context->IncSequenceId(); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { + errCode = PushStart(context); + } else { + errCode = PullResponseStart(context); + } + if ((mode == SyncModeType::PUSH_AND_PULL) && errCode == -E_EKEYREVOKED) { + LOGE("[DataSync] wait for recv finished,label=%s,device=%s", label_.c_str(), STR_MASK(deviceId_)); + isAllDataHasSent_ = true; + return -E_EKEYREVOKED; + } + if (context->IsSkipTimeoutError(errCode)) { + // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. + // just return to avoid higher pressure for send. + return E_OK; + } + if (errCode != E_OK) { + LOGE("[DataSync] InnerSend errCode=%d", errCode); + return errCode; + } + } + return E_OK; +} + +void SingleVerDataSync::InnerClearSyncStatus() +{ + sessionId_ = 0; + reSendMap_.clear(); + windowSize_ = 0; + maxSequenceIdHasSent_ = 0; + isAllDataHasSent_ = false; +} + +int SingleVerDataSync::TryContinueSync(SingleVerSyncTaskContext *context, const Message *message) +{ + if (message == nullptr) { + LOGE("[DataSync] AckRecv message nullptr"); + return -E_INVALID_ARGS; + } + const DataAckPacket *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value + uint32_t sessionId = message->GetSessionId(); + uint32_t sequenceId = message->GetSequenceId(); + + std::lock_guard lock(lock_); + LOGI("[DataSync] recv ack seqId=%d,packetId=%llu,winSize=%d,label=%s,dev=%s", sequenceId, packetId, windowSize_, + label_.c_str(), STR_MASK(deviceId_)); + if (sessionId != sessionId_) { + LOGI("[DataSync] ignore ack,sessionId is different"); + return E_OK; + } + if (reSendMap_.count(sequenceId) != 0) { + reSendMap_.erase(sequenceId); + windowSize_++; + } else { + LOGI("[DataSync] ack seqId not in map"); + return E_OK; + } + if (!isAllDataHasSent_) { + return InnerSyncStart(context); + } else if (reSendMap_.size() == 0) { + context->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); + InnerClearSyncStatus(); + return -E_FINISHED; + } + return E_OK; +} + +void SingleVerDataSync::ClearSyncStatus() +{ + std::lock_guard lock(lock_); + InnerClearSyncStatus(); +} + +int SingleVerDataSync::ReSendData(SingleVerSyncTaskContext *context) +{ + if (reSendMap_.empty()) { + LOGI("[DataSync] ReSend map empty"); + return -E_INTERNAL_ERROR; + } + uint32_t sequenceId = reSendMap_.begin()->first; + ReSendInfo reSendInfo = reSendMap_.begin()->second; + LOGI("[DataSync] ReSend mode=%d,start=%llu,end=%llu,delStart=%llu,delEnd=%llu,seqId=%d,packetId=%llu,windowsize=%d," + "label=%s,deviceId=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, sequenceId, reSendInfo.packetId, windowSize_, label_.c_str(), STR_MASK(deviceId_)); + DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, + reSendInfo.deleteBeginTime, reSendInfo.deleteEndTime, reSendInfo.packetId}; + return ReSend(context, dataReSendInfo); +} + std::string SingleVerDataSync::GetLocalDeviceName() { std::string deviceInfo; @@ -252,6 +395,26 @@ int SingleVerDataSync::SaveData(const SingleVerSyncTaskContext *context, const s return errCode; } +void SingleVerDataSync::ResetSyncStatus(int inMode, SingleVerSyncTaskContext *context) +{ + mode_ = inMode; + maxSequenceIdHasSent_ = 0; + isAllDataHasSent_ = false; + context->ReSetSequenceId(); + reSendMap_.clear(); + if (context->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { + windowSize_ = LOW_VERSION_WINDOW_SIZE; + } else { + windowSize_ = HIGH_VERSION_WINDOW_SIZE; + } + int mode = SyncOperation::TransferSyncMode(inMode); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL) { + sessionId_ = context->GetRequestSessionId(); + } else { + sessionId_ = context->GetResponseSessionId(); + } +} + TimeStamp SingleVerDataSync::GetMaxSendDataTime(const std::vector &inData) { TimeStamp stamp = 0; @@ -574,6 +737,38 @@ void SingleVerDataSync::TransSendDataItemToLocal(const SingleVerSyncTaskContext } } +void SingleVerDataSync::SetSessionEndTimeStamp(TimeStamp end) +{ + sessionEndTimeStamp_ = end; +} + +TimeStamp SingleVerDataSync::GetSessionEndTimeStamp() const +{ + return sessionEndTimeStamp_; +} + +void SingleVerDataSync::UpdateSendInfo(SyncTimeRange dataTimeRange, SingleVerSyncTaskContext *context) +{ + ReSendInfo reSendInfo; + reSendInfo.start = dataTimeRange.beginTime; + reSendInfo.end = dataTimeRange.endTime; + reSendInfo.deleteBeginTime = dataTimeRange.deleteBeginTime; + reSendInfo.deleteEndTime = dataTimeRange.deleteEndTime; + reSendInfo.packetId = context->GetPacketId(); + maxSequenceIdHasSent_++; + reSendMap_[maxSequenceIdHasSent_] = reSendInfo; + windowSize_--; + ContinueToken token; + context->GetContinueToken(token); + if (token == nullptr) { + isAllDataHasSent_ = true; + } + LOGI("[DataSync] mode=%d,start=%llu,end=%llu,deleteStart=%llu,deleteEnd=%llu,seqId=%d,packetId=%llu,window_size=%d," + "isAllSend=%d,label=%s,device=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, maxSequenceIdHasSent_, reSendInfo.packetId, windowSize_, isAllDataHasSent_, + label_.c_str(), STR_MASK(deviceId_)); +} + void SingleVerDataSync::FillDataRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, SyncEntry &syncData, int sendCode, int mode) { @@ -654,9 +849,8 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; UpdateWaterMark isUpdateWaterMark; SyncTimeRange dataTime = GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); - context->SetSequenceStartAndEndTimeStamp(dataTime); if (errCode == E_OK) { - context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); + SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } FillDataRequestPacket(packet, context, syncData, errCode, mode); errCode = SendDataPacket(curType, packet, context); @@ -664,6 +858,9 @@ int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) if (performance != nullptr) { performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_MACHINE_START_TO_PUSH_SEND); } + if (errCode == E_OK || errCode == -E_TIMEOUT) { + UpdateSendInfo(dataTime, context); + } if (errCode == E_OK) { if (curType == SyncType::QUERY_SYNC_TYPE && (context->GetQuery().HasLimit() || context->GetQuery().HasOrderBy())) { @@ -723,7 +920,6 @@ int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); WaterMark endMark = context->GetEndMark(); SyncTimeRange dataTime = {localMark, deleteMark, localMark, deleteMark}; - context->SetSequenceStartAndEndTimeStamp(dataTime); packet->SetBasicInfo(E_OK, version, context->GetMode()); packet->SetWaterMark(localMark, peerMark, deleteMark); @@ -736,6 +932,7 @@ int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) LOGD("[DataSync][Pull] curType=%d,local=%llu,del=%llu,end=%llu,peer=%llu,label=%s,dev=%s", syncType, localMark, deleteMark, peerMark, endMark, label_.c_str(), STR_MASK(GetDeviceId())); + UpdateSendInfo(dataTime, context); return SendDataPacket(syncType, packet, context); } @@ -763,13 +960,14 @@ int SingleVerDataSync::PullResponseStart(SingleVerSyncTaskContext *context) } SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; UpdateWaterMark isUpdateWaterMark; - SyncTimeRange dataTime = - GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); - context->SetSequenceStartAndEndTimeStamp(dataTime); + SyncTimeRange dataTime = GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); if (errCode == E_OK) { - context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); + SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } errCode = SendPullResponseDataPkt(ackCode, syncData, context); + if (errCode == E_OK || errCode == -E_TIMEOUT) { + UpdateSendInfo(dataTime, context); + } if (errCode == E_OK) { if (curType == SyncType::QUERY_SYNC_TYPE && (context->GetQuery().HasLimit() || context->GetQuery().HasOrderBy())) { @@ -1016,6 +1214,11 @@ int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, SyncEntry &syncOutDa return errCode; } +void SingleVerDataSync::SendFinishedDataAck(SingleVerSyncTaskContext *context, const Message *message) +{ + SendDataAck(context, message, E_OK, 0); +} + int SingleVerDataSync::SendDataAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, WaterMark maxSendDataTime) { @@ -1048,6 +1251,32 @@ int SingleVerDataSync::SendDataAck(SingleVerSyncTaskContext *context, const Mess return errCode; } +bool SingleVerDataSync::AckPacketIdCheck(const Message *message) +{ + if (message == nullptr) { + LOGE("[DataSync] AckRecv message nullptr"); + return false; + } + if (message->GetMessageType() == TYPE_NOTIFY || message->IsFeedbackError()) { + return false; + } + const DataAckPacket *packet = message->GetObject(); + if (packet == nullptr) { + return false; + } + uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value + std::lock_guard lock(lock_); + uint32_t sequenceId = message->GetSequenceId(); + if (reSendMap_.count(sequenceId) != 0) { + uint64_t originalPacketId = reSendMap_[sequenceId].packetId; + if (DataAckPacket::IsPacketIdValid(packetId) && packetId != originalPacketId) { + LOGE("[DataSync] packetId[%llu] is not match with original[%llu]", packetId, originalPacketId); + return false; + } + } + return true; +} + int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message *message) { int errCode = AckMsgErrnoCheck(context, message); @@ -1067,8 +1296,7 @@ int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message } if (recvCode == -E_NEED_ABILITY_SYNC || recvCode == -E_NOT_PERMIT) { - // after set sliding window sender err, we can ReleaseContinueToken, avoid crash - context->SetSlidingWindowSenderErr(true); + // we should ReleaseContinueToken, avoid crash LOGI("[DataSync][AckRecv] Data sync abort,recvCode =%d,label =%s,dev=%s", recvCode, label_.c_str(), STR_MASK(GetDeviceId())); context->ReleaseContinueToken(); @@ -1158,9 +1386,6 @@ void SingleVerDataSync::GetPullEndWatermark(const SingleVerSyncTaskContext *cont int SingleVerDataSync::DealWaterMarkException(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, const std::vector &reserved) { - // after set sliding window sender err, we can SaveLocalWaterMark, avoid sliding window sender re save a wrong - // waterMark again. - context->SetSlidingWindowSenderErr(true); WaterMark deletedWaterMark = 0; SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; if (curType == SyncType::QUERY_SYNC_TYPE) { @@ -1660,12 +1885,12 @@ void SingleVerDataSync::FillRequestReSendPacket(const SingleVerSyncTaskContext * // transer reSend mode, RESPONSE_PULL transfer to push or query push // PUSH_AND_PULL mode which sequenceId lager than first transfer to push or query push int reSendMode = GetReSendMode(context->GetMode(), reSendInfo.sequenceId, curType); - if (context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) || + if (GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) || SyncOperation::TransferSyncMode(context->GetMode()) == SyncModeType::PULL) { LOGI("[DataSync][ReSend] set lastid,label=%s,dev=%s", label_.c_str(), STR_MASK(GetDeviceId())); packet->SetLastSequence(); } - if (sendCode == E_OK && context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) && + if (sendCode == E_OK && GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) && context->GetMode() == SyncModeType::RESPONSE_PULL) { sendCode = SEND_FINISHED; } @@ -2148,4 +2373,30 @@ std::string SingleVerDataSync::GetDeleteSyncId(const SingleVerSyncTaskContext *c #endif return id; } + +void SingleVerDataSync::PutDataMsg(Message *message) +{ + return msgSchedule_.PutMsg(message); +} + +Message *SingleVerDataSync::MoveNextDataMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, + bool &isNeedContinue) +{ + return msgSchedule_.MoveNextMsg(context, isNeedHandle, isNeedContinue); +} + +bool SingleVerDataSync::IsNeedReloadQueue() +{ + return msgSchedule_.IsNeedReloadQueue(); +} + +void SingleVerDataSync::ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, const Message *message) +{ + msgSchedule_.ScheduleInfoHandle(isNeedHandleStatus, isNeedClearMap, message); +} + +void SingleVerDataSync::ClearDataMsg() +{ + msgSchedule_.ClearMsg(); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h index bcb2b0b973954280de5e408e9bd3c02c4ad3acac..8f0e4b023391ce7cb75d5ddd473e1498eff28daf 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h @@ -20,6 +20,7 @@ #include "isync_interface.h" #include "meta_data.h" #include "parcel.h" +#include "single_ver_data_message_schedule.h" #include "single_ver_data_packet.h" #include "single_ver_kvdb_sync_interface.h" #include "single_ver_sync_task_context.h" @@ -29,6 +30,15 @@ namespace DistributedDB { using SendDataItem = SingleVerKvEntry *; +struct ReSendInfo { + TimeStamp start = 0; + TimeStamp end = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp deleteEndTime = 0; + // packetId is used for matched ackpacket packetId which saved in ackPacket.reserve + // if equaled, means need to handle the ack, or drop. it is always increased + uint64_t packetId = 0; +}; struct DataSyncReSendInfo { uint32_t sessionId = 0; @@ -54,6 +64,12 @@ public: int Initialize(ISyncInterface *inStorage, ICommunicator *inCommunicateHandle, std::shared_ptr &inMetadata, const std::string &deviceId); + + int SyncStart(int mode, SingleVerSyncTaskContext *context); + + int TryContinueSync(SingleVerSyncTaskContext *context, const Message *message); + + void ClearSyncStatus(); int PushStart(SingleVerSyncTaskContext *context); @@ -65,6 +81,8 @@ public: int DataRequestRecv(SingleVerSyncTaskContext *context, const Message *message, WaterMark &pullEndWatermark); + bool AckPacketIdCheck(const Message *message); + int AckRecv(SingleVerSyncTaskContext *context, const Message *message); void SendSaveDataNotifyPacket(SingleVerSyncTaskContext *context, uint32_t pktVersion, uint32_t sessionId, @@ -73,8 +91,6 @@ public: virtual int SendDataAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, WaterMark maxSendDataTime); - int32_t ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo); - int CheckPermitSendData(int inMode, SingleVerSyncTaskContext *context); std::string GetLabel() const; @@ -94,6 +110,18 @@ public: void ControlAckErrorHandle(const SingleVerSyncTaskContext *context, const std::shared_ptr &subManager) const; + void PutDataMsg(Message *message); + + Message *MoveNextDataMsg(SingleVerSyncTaskContext *context, bool &isNeedHandle, bool &isNeedContinue); + + bool IsNeedReloadQueue(); + + void SendFinishedDataAck(SingleVerSyncTaskContext *context, const Message *message); + + void ScheduleInfoHandle(bool isNeedHandleStatus, bool isNeedClearMap, const Message *message); + + void ClearDataMsg(); + protected: static const int SEND_FINISHED = 0xff; static const int LOCAL_WATER_MARK_NOT_INIT = 0xaa; @@ -101,10 +129,24 @@ protected: static const int WATER_MARK_INVALID = 0xbb; static const int MTU_SIZE = 28311552; // 27MB + void ResetSyncStatus(int inMode, SingleVerSyncTaskContext *context); + + int InnerSyncStart(SingleVerSyncTaskContext *context); + + void InnerClearSyncStatus(); + + int ReSendData(SingleVerSyncTaskContext *context); + + int32_t ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo); + TimeStamp GetMaxSendDataTime(const std::vector &inData); TimeStamp GetMinSendDataTime(const std::vector &inData, WaterMark localMark); + void SetSessionEndTimeStamp(TimeStamp end); + + TimeStamp GetSessionEndTimeStamp() const; + void FillDataRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, SyncEntry &syncData, int sendCode, int mode); @@ -220,6 +262,8 @@ protected: int GetReSendMode(int mode, uint32_t sequenceId, SyncType syncType); + void UpdateSendInfo(SyncTimeRange dataTimeRange, SingleVerSyncTaskContext *context); + void FillRequestReSendPacket(const SingleVerSyncTaskContext *context, DataRequestPacket *packet, DataSyncReSendInfo reSendInfo, SyncEntry &syncData, int sendCode); @@ -260,8 +304,25 @@ protected: std::shared_ptr metadata_; std::string label_; std::string deviceId_; + + SingleVerDataMessageSchedule msgSchedule_; + + static const int HIGH_VERSION_WINDOW_SIZE = 3; + static const int LOW_VERSION_WINDOW_SIZE = 1; + // below param is about sliding sync info, is different from every sync task + std::mutex lock_; + int mode_ = 0; // sync mode, may diff from context mode if trigger pull_response while push finish + uint32_t sessionId_ = 0; + // sequenceId as key + std::map reSendMap_; + // remaining sending window + int32_t windowSize_ = 0; + // max sequenceId has been sent + uint32_t maxSequenceIdHasSent_ = 0; + bool isAllDataHasSent_ = false; + // in a sync session, the last data timeStamp + TimeStamp sessionEndTimeStamp_ = 0; }; } // namespace DistributedDB #endif // SINGLE_VER_DATA_SYNC_NEW_H - diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp deleted file mode 100755 index 4b9926ac647d16271b4a86a463484871d96eef60..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "single_ver_data_sync_with_sliding_window.h" - -namespace DistributedDB { -int SingleVerDataSyncWithSlidingWindow::SenderStart(int32_t mode, SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - return sender_.SendStart(mode, context, dataSync); -} - -int SingleVerDataSyncWithSlidingWindow::SenderAckRecv(const Message *message) -{ - return sender_.AckRecv(message); -} - -int SingleVerDataSyncWithSlidingWindow::PreHandleSenderAckRecv(const Message *message) -{ - return sender_.PreHandleAckRecv(message); -} - -void SingleVerDataSyncWithSlidingWindow::SenderClear() -{ - sender_.Clear(); -} - -void SingleVerDataSyncWithSlidingWindow::SetSenderErr(bool isErr) -{ - sender_.SetErr(isErr); -} - -int SingleVerDataSyncWithSlidingWindow::ReceiverInit(SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - return receiver_.Initialize(context, dataSync); -} - -int SingleVerDataSyncWithSlidingWindow::SenderInit(SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - return sender_.Initialize(context, dataSync); -} - -void SingleVerDataSyncWithSlidingWindow::ReceiverClear() -{ - receiver_.Clear(); -} - -int SingleVerDataSyncWithSlidingWindow::Receive(Message *inMsg) -{ - return receiver_.Receive(inMsg); -} -} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp index 31fc0d11482587a21a8e033ea65ef83c591bf528..df15a8dab52d3655169aaef0eec4df26a83edd77 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp @@ -1,154 +1,153 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "single_ver_relational_syncer.h" -#ifdef RELATIONAL_STORE -#include "db_common.h" -#include "relational_db_sync_interface.h" -#include "single_ver_sync_engine.h" - -namespace DistributedDB { -int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) -{ - const auto &syncInterface = static_cast(syncInterface_); - std::vector tablesQuery; - if (param.isQuerySync) { - tablesQuery.push_back(param.syncQuery); - } else { - tablesQuery = syncInterface->GetTablesQuery(); - } - std::set subSyncIdSet; - int errCode = GenerateEachSyncTask(param, syncId, tablesQuery, subSyncIdSet); - if (errCode != E_OK) { - DoRollBack(subSyncIdSet); - return errCode; - } - return E_OK; -} - -int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, - const std::vector &tablesQuery, std::set &subSyncIdSet) -{ - SyncParma subParam = param; - subParam.isQuerySync = true; - int errCode = E_OK; - for (const QuerySyncObject &table : tablesQuery) { - uint32_t subSyncId = GenerateSyncId(); - LOGI("[SingleVerRelationalSyncer] SubSyncId %d create by SyncId %d, tableName = %s", - subSyncId, syncId, table.GetRelationTableName().c_str()); - subParam.syncQuery = table; - subParam.onComplete = std::bind(&SingleVerRelationalSyncer::DoOnSubSyncComplete, this, subSyncId, - syncId, param, std::placeholders::_1); - { - std::lock_guard lockGuard(syncMapLock_); - fullSyncIdMap_[syncId].insert(subSyncId); - } - errCode = GenericSyncer::PrepareSync(subParam, subSyncId); - if (errCode != E_OK) { - LOGW("[SingleVerRelationalSyncer] PrepareSync failed errCode:%d", errCode); - std::lock_guard lockGuard(syncMapLock_); - fullSyncIdMap_[syncId].erase(subSyncId); - break; - } - subSyncIdSet.insert(subSyncId); - } - return errCode; -} - -void SingleVerRelationalSyncer::DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, - const SyncParma ¶m, const std::map &devicesMap) -{ - static std::map statusMap = { - { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, - { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, - { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, - { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, - { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, - { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, - { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, - { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, - { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, - { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, - { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, - }; - bool allFinish = true; - { - std::lock_guard lockGuard(syncMapLock_); - fullSyncIdMap_[syncId].erase(subSyncId); - allFinish = fullSyncIdMap_[syncId].empty(); - for (const auto &item : devicesMap) { - DBStatus status = DB_ERROR; - if (statusMap.find(item.second) != statusMap.end()) { - status = statusMap[item.second]; - } - resMap_[syncId][item.first][param.syncQuery.GetRelationTableName()] = status; - } - } - if (allFinish) { - DoOnComplete(param, syncId); - } -} - -void SingleVerRelationalSyncer::DoRollBack(std::set &subSyncIdSet) -{ - for (const auto &removeId : subSyncIdSet) { - int retCode = RemoveSyncOperation((int)removeId); - if (retCode != E_OK) { - LOGW("[SingleVerRelationalSyncer] RemoveSyncOperation failed errCode:%d, syncId:%d", retCode, removeId); - } - } -} - -void SingleVerRelationalSyncer::DoOnComplete(const SyncParma ¶m, uint32_t syncId) -{ - if (!param.relationOnComplete) { - return; - } - std::map> syncRes; - std::map> tmpMap; - { - std::lock_guard lockGuard(syncMapLock_); - tmpMap = resMap_[syncId]; - } - for (const auto &devicesRes : tmpMap) { - for (const auto &tableRes : devicesRes.second) { - syncRes[devicesRes.first].push_back( - {tableRes.first, static_cast(tableRes.second)}); - } - } - param.relationOnComplete(syncRes); - { - std::lock_guard lockGuard(syncMapLock_); - resMap_.erase(syncId); - } -} - -void SingleVerRelationalSyncer::EnableAutoSync(bool enable) -{ -} - -int SingleVerRelationalSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) -{ - return E_OK; -} - -void SingleVerRelationalSyncer::LocalDataChanged(int notifyEvent) -{ -} - -void SingleVerRelationalSyncer::RemoteDataChanged(const std::string &device) -{ -} -} +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "single_ver_relational_syncer.h" +#ifdef RELATIONAL_STORE +#include "db_common.h" +#include "relational_db_sync_interface.h" +#include "single_ver_sync_engine.h" + +namespace DistributedDB { +int SingleVerRelationalSyncer::Sync(const SyncParma ¶m) +{ + if (param.mode == SYNC_MODE_PUSH_PULL) { + return -E_NOT_SUPPORT; + } + if (param.syncQuery.GetRelationTableName().empty()) { + return -E_NOT_SUPPORT; + } + return GenericSyncer::Sync(param); +} + +int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) +{ + const auto &syncInterface = static_cast(syncInterface_); + std::vector tablesQuery; + if (param.isQuerySync) { + tablesQuery.push_back(param.syncQuery); + } else { + tablesQuery = syncInterface->GetTablesQuery(); + } + std::set subSyncIdSet; + int errCode = GenerateEachSyncTask(param, syncId, tablesQuery, subSyncIdSet); + if (errCode != E_OK) { + DoRollBack(subSyncIdSet); + return errCode; + } + if (param.wait) { + DoOnComplete(param, syncId); + } + return E_OK; +} + +int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, + const std::vector &tablesQuery, std::set &subSyncIdSet) +{ + SyncParma subParam = param; + subParam.isQuerySync = true; + int errCode = E_OK; + for (const QuerySyncObject &table : tablesQuery) { + uint32_t subSyncId = GenerateSyncId(); + LOGI("[SingleVerRelationalSyncer] SubSyncId %d create by SyncId %d, tableName = %s", + subSyncId, syncId, table.GetRelationTableName().c_str()); + subParam.syncQuery = table; + subParam.onComplete = std::bind(&SingleVerRelationalSyncer::DoOnSubSyncComplete, this, subSyncId, + syncId, param, std::placeholders::_1); + { + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].insert(subSyncId); + } + errCode = GenericSyncer::PrepareSync(subParam, subSyncId); + if (errCode != E_OK) { + LOGW("[SingleVerRelationalSyncer] PrepareSync failed errCode:%d", errCode); + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].erase(subSyncId); + break; + } + subSyncIdSet.insert(subSyncId); + } + return errCode; +} + +void SingleVerRelationalSyncer::DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, + const SyncParma ¶m, const std::map &devicesMap) +{ + bool allFinish = true; + { + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].erase(subSyncId); + allFinish = fullSyncIdMap_[syncId].empty(); + for (const auto &item : devicesMap) { + resMap_[syncId][item.first][param.syncQuery.GetRelationTableName()] = static_cast(item.second); + } + } + // block sync do callback in sync function + if (allFinish && !param.wait) { + DoOnComplete(param, syncId); + } +} + +void SingleVerRelationalSyncer::DoRollBack(std::set &subSyncIdSet) +{ + for (const auto &removeId : subSyncIdSet) { + int retCode = RemoveSyncOperation((int)removeId); + if (retCode != E_OK) { + LOGW("[SingleVerRelationalSyncer] RemoveSyncOperation failed errCode:%d, syncId:%d", retCode, removeId); + } + } +} + +void SingleVerRelationalSyncer::DoOnComplete(const SyncParma ¶m, uint32_t syncId) +{ + if (!param.relationOnComplete) { + return; + } + std::map> syncRes; + std::map> tmpMap; + { + std::lock_guard lockGuard(syncMapLock_); + tmpMap = resMap_[syncId]; + } + for (const auto &devicesRes : tmpMap) { + for (const auto &tableRes : devicesRes.second) { + syncRes[devicesRes.first].push_back( + {tableRes.first, static_cast(tableRes.second)}); + } + } + param.relationOnComplete(syncRes); + { + std::lock_guard lockGuard(syncMapLock_); + resMap_.erase(syncId); + fullSyncIdMap_.erase(syncId); + } +} + +void SingleVerRelationalSyncer::EnableAutoSync(bool enable) +{ +} + +int SingleVerRelationalSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +{ + return E_OK; +} + +void SingleVerRelationalSyncer::LocalDataChanged(int notifyEvent) +{ +} + +void SingleVerRelationalSyncer::RemoteDataChanged(const std::string &device) +{ +} +} #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h index 159a011eaadaae1938b108c1be16add91083fc17..15e4ef814348398e2d277300316edc6abf7bdea9 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h @@ -22,6 +22,9 @@ public: SingleVerRelationalSyncer() = default; ~SingleVerRelationalSyncer() override = default; + // Sync function. use SyncParma to reduce paramter. + int Sync(const SyncParma ¶m) override; + void EnableAutoSync(bool enable) override; int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; @@ -50,6 +53,5 @@ private: std::map>> resMap_; }; } - #endif -#endif // RELATIONAL_SYNCER_H +#endif // RELATIONAL_SYNCER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp deleted file mode 100644 index ea78cbe1b0f044cb39ed574c1388b11c07a7f03c..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "single_ver_subscribe_manager.h" -#include "db_common.h" -#include "sync_types.h" - -namespace DistributedDB { -void SingleVerSubscribeManager::ClearRemoteSubscribeQuery(const std::string &device) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - ClearSubscribeQuery(device, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::ClearAllRemoteQuery() -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - remoteSubscribedMap_.clear(); - remoteSubscribedTotalMap_.clear(); -} - -void SingleVerSubscribeManager::ClearLocalSubscribeQuery(const std::string &device) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - unFinishedLocalAutoSubMap_.erase(device); - ClearSubscribeQuery(device, localSubscribeMap_, localSubscribeTotalMap_); -} - -int SingleVerSubscribeManager::ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - int errCode = ReserveSubscribeQuery(device, query, remoteSubscribedMap_, remoteSubscribedTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote reserve err=%d", - STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - int errCode = ActiveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote active err=%d", - STR_MASK(device), STR_MASK(queryId), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - int errCode = ReserveSubscribeQuery(device, query, localSubscribeMap_, localSubscribeTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local reserve err=%d", - STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); - return errCode; -} - -int SingleVerSubscribeManager::ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - int errCode = ActiveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local active err=%d", - STR_MASK(device), STR_MASK(queryId), errCode); - if (errCode != E_OK) { - return errCode; - } - if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && - unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { - unFinishedLocalAutoSubMap_[device].erase(queryId); - } - return errCode; -} - -void SingleVerSubscribeManager::DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - DeleteSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); -} - -void SingleVerSubscribeManager::DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - DeleteSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::PutLocalUnFiniedSubQueries(const std::string &device, - std::vector &subscribeQueries) -{ - LOGI("[SingleVerSubscribeManager] put local unfinished subscribe queries, nums=%d", subscribeQueries.size()); - std::unique_lock lockGuard(localSubscribeMapLock_); - if (subscribeQueries.size() == 0) { - unFinishedLocalAutoSubMap_.erase(device); - return; - } - unFinishedLocalAutoSubMap_[device].clear(); - auto iter = unFinishedLocalAutoSubMap_.find(device); - for (const auto &query : subscribeQueries) { - iter->second.insert(query.GetIdentify()); - } -} - -void SingleVerSubscribeManager::GetAllUnFinishSubQueries( - std::map> &allSyncQueries) const -{ - std::shared_lock lock(localSubscribeMapLock_); - for (auto &item : unFinishedLocalAutoSubMap_) { - if (item.second.size() == 0) { - continue; - } - allSyncQueries[item.first] = {}; - auto iter = allSyncQueries.find(item.first); - for (const auto &queryId : item.second) { - auto iterTmp = localSubscribeTotalMap_.find(queryId); - if (iterTmp == localSubscribeTotalMap_.end()) { - LOGI("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryId)); - continue; - } - iter->second.push_back(iterTmp->second.first); - } - } -} - -void SingleVerSubscribeManager::RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(remoteSubscribedMapLock_); - std::string queryId = query.GetIdentify(); - RemoveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); -} - -void SingleVerSubscribeManager::RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) -{ - std::unique_lock lockGuard(localSubscribeMapLock_); - std::string queryId = query.GetIdentify(); - RemoveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); - if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && - unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { - unFinishedLocalAutoSubMap_[device].erase(queryId); - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s delete from UnFinishedMap", - STR_MASK(device), STR_MASK(queryId)); - if (unFinishedLocalAutoSubMap_[device].size() == 0) { - LOGI("[SingleVerSubscribeManager] dev=%s delete from unFinish map", STR_MASK(device)); - unFinishedLocalAutoSubMap_.erase(device); - } - } -} - -void SingleVerSubscribeManager::GetLocalSubscribeQueries(const std::string &device, - std::vector &subscribeQueries) const -{ - std::shared_lock lock(localSubscribeMapLock_); - GetSubscribeQueries(device, localSubscribeMap_, localSubscribeTotalMap_, subscribeQueries); -} - -void SingleVerSubscribeManager::GetRemoteSubscribeQueries(const std::string &device, - std::vector &subscribeQueries) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - GetSubscribeQueries(device, remoteSubscribedMap_, remoteSubscribedTotalMap_, subscribeQueries); -} - -bool SingleVerSubscribeManager::IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - auto iter = remoteSubscribedMap_.find(device); - if (iter == remoteSubscribedMap_.end()) { - LOGD("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); - return false; - } - std::string queryId = query.GetIdentify(); - auto subIter = iter->second.find(queryId); - if (subIter == iter->second.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryId)); - return false; - } - return true; -} - -void SingleVerSubscribeManager::GetRemoteSubscribeQueryIds(const std::string &device, - std::vector &subscribeQueryIds) const -{ - std::shared_lock lockGuard(remoteSubscribedMapLock_); - auto iter = remoteSubscribedMap_.find(device); - if (iter == remoteSubscribedMap_.end()) { - LOGI("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : iter->second) { - if (remoteSubscribedTotalMap_.find(queryInfo.first) == remoteSubscribedTotalMap_.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryInfo.first)); - continue; - } - subscribeQueryIds.push_back(queryInfo.first); - } -} - -int SingleVerSubscribeManager::LocalSubscribeLimitCheck(const std::vector &devices, - QuerySyncObject &query) const -{ - std::shared_lock lock(localSubscribeMapLock_); - int devNum = localSubscribeMap_.size(); - for (const auto &device : devices) { - if (localSubscribeMap_.find(device) != localSubscribeMap_.end()) { - continue; - } - devNum++; - if (devNum > MAX_DEVICES_NUM) { - LOGE("[SingleVerSubscribeManager] local subscribe devices is over limit"); - return -E_MAX_LIMITS; - } - } - std::string queryId = query.GetIdentify(); - auto allIter = localSubscribeTotalMap_.find(queryId); - if (allIter == localSubscribeTotalMap_.end() && localSubscribeTotalMap_.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { - LOGE("[SingleVerSubscribeManager] all local subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - return E_OK; -} - -void SingleVerSubscribeManager::ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGI("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : subscribeMap[device]) { - if (subscribedTotalMap.find(queryInfo.first) != subscribedTotalMap.end()) { - if (subscribedTotalMap[queryInfo.first].second > 0) { - subscribedTotalMap[queryInfo.first].second--; - } - if (subscribedTotalMap[queryInfo.first].second == 0) { - LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryInfo.first)); - subscribedTotalMap.erase(queryInfo.first); - } - } - } - subscribeMap.erase(device); - LOGI("[SingleVerSubscribeManager] clear dev=%s` remote subscribe queies finished", STR_MASK(device)); -} - -int SingleVerSubscribeManager::ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - std::string queryId = query.GetIdentify(); - auto iter = subscribeMap.find(device); - auto allIter = subscribedTotalMap.find(queryId); - // limit check - if (allIter == subscribedTotalMap.end() && subscribedTotalMap.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { - LOGE("[SingleVerSubscribeManager] all subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - if (iter == subscribeMap.end() && subscribeMap.size() >= MAX_DEVICES_NUM) { - LOGE("[SingleVerSubscribeManager] subscribe devices is over limit"); - return -E_MAX_LIMITS; - } - if (iter != subscribeMap.end() && iter->second.find(queryId) == iter->second.end() && - iter->second.size() >= MAX_SUBSCRIBE_NUM_PER_DEV) { - LOGE("[SingleVerSubscribeManager] subscribe sums is over limit"); - return -E_MAX_LIMITS; - } - if (iter != subscribeMap.end() && iter->second.find(queryId) != iter->second.end() && - iter->second[queryId] == SubscribeStatus::ACTIVE) { - LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s already active in map", - STR_MASK(device), STR_MASK(queryId)); - return E_OK; - } - - if (iter == subscribeMap.end()) { - subscribeMap[device] = std::map {}; - } - bool isNeedInc = false; - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - subscribeMap[device][queryId] = SubscribeStatus::NOT_ACTIVE; - isNeedInc = true; - } - if (allIter == subscribedTotalMap.end()) { - subscribedTotalMap[queryId] = {query, 1}; - } else if (isNeedInc) { - subscribedTotalMap[queryId].second++; - } - return E_OK; -} - -int SingleVerSubscribeManager::ActiveSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribedTotalMap.find(queryId) == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); - return -E_INTERNAL_ERROR; - } - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s in localSubscribeMap", STR_MASK(device)); - return -E_INTERNAL_ERROR; - } - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); - return -E_INTERNAL_ERROR; - } - subscribeMap[device][queryId] = SubscribeStatus::ACTIVE; - return E_OK; -} - -void SingleVerSubscribeManager::DeleteSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - if (subscribeMap.find(device) == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s in map", STR_MASK(device)); - return; - } - if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { - LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); - return; - } - SubscribeStatus queryStatus = subscribeMap[device][queryId]; - // not permit to delete the query when something wrong this time,because it is subscribed successfully last time - if (queryStatus == SubscribeStatus::ACTIVE) { - LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s is active, no need to del", - STR_MASK(device), STR_MASK(queryId)); - return; - } - subscribeMap[device].erase(queryId); - auto iter = subscribedTotalMap.find(queryId); - if (iter == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); - return; - } - iter->second.second--; - if (iter->second.second <= 0) { - LOGI("[SingleVerSubscribeManager] del queryId=%s from SubscribeTotalMap", STR_MASK(queryId)); - subscribedTotalMap.erase(queryId); - } - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribeMap success", - STR_MASK(device), STR_MASK(queryId)); -} - -void SingleVerSubscribeManager::RemoveSubscribeQuery(const std::string &device, const std::string &queryId, - SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) -{ - auto iter = subscribeMap.find(device); - if (iter == subscribeMap.end()) { - LOGE("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); - return; - } - if (iter->second.find(queryId) == subscribeMap[device].end()) { - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s not in SubscribedMap", STR_MASK(device), STR_MASK(queryId)); - return; - } - iter->second.erase(queryId); - auto allIter = subscribedTotalMap.find(queryId); - if (allIter == subscribedTotalMap.end()) { - LOGI("[SingleVerSubscribeManager] queryId=%s not in TotalMap", STR_MASK(queryId)); - return; - } - allIter->second.second--; - if (allIter->second.second <= 0) { - subscribedTotalMap.erase(queryId); - LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryId)); - } - LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribedMap success", - STR_MASK(device), STR_MASK(queryId)); -} - -void SingleVerSubscribeManager::GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, - const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const -{ - auto iter = subscribeMap.find(device); - if (iter == subscribeMap.end()) { - LOGD("[SingleVerSubscribeManager] dev=%s not in localSubscribeMap", STR_MASK(device)); - return; - } - for (const auto &queryInfo : iter->second) { - auto iterTmp = subscribedTotalMap.find(queryInfo.first); - if (iterTmp == subscribedTotalMap.end()) { - LOGE("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryInfo.first)); - continue; - } - subscribeQueries.push_back(iterTmp->second.first); - } -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h deleted file mode 100644 index c152e0f29e0f966436f3e8c2d048add4aa3e6f2c..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SINGLE_VER_SUBSCRIBE_MANAGER_H -#define SINGLE_VER_SUBSCRIBE_MANAGER_H - -#include -#include -#include "query_sync_object.h" - -namespace DistributedDB { -enum class SubscribeStatus { - NOT_ACTIVE = 0, - ACTIVE = 1, -}; - -using SubscribeMap = std::map>; -using SubscribedTotalMap = std::map>; - -class SingleVerSubscribeManager { -public: - SingleVerSubscribeManager() = default; - ~SingleVerSubscribeManager() {}; - - DISABLE_COPY_ASSIGN_MOVE(SingleVerSubscribeManager); - - // clear remoteSubscribeMap_[device] list when remote db is closed or dev offline. - void ClearRemoteSubscribeQuery(const std::string &device); - - // clear localSubscribeMap_[device] list when device is offline. - void ClearLocalSubscribeQuery(const std::string &device); - - void ClearAllRemoteQuery(); - - // add query when receive subscribe command - int ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // active query to ACTIVE when send ack ok - int ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // reserve query when user call SubscribeRemoteQuery, status set to NOT_ACTIVE - int ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // active query to ACTIVE when receive ack ok - int ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // delete local subscribe query when recv wrong errCode, only not_active status allowed to del - void DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // delete remote subscribe query when send msg failed, only not_active status allowed to del - void DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // put subscribe queries into unfinished map when remote db online - void PutLocalUnFiniedSubQueries(const std::string &device, std::vector &subscribeQueries); - - // get all device unFinished subscribe queries which triggered by auto subscribe and need retry subscribe - void GetAllUnFinishSubQueries(std::map> &allSyncQueries) const; - - // remove query when receive unsubscribe command - void RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // remove query when user call UnSubscribeRemoteQuery - void RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); - - // get device active subscribeQueries from localSubscribeMap_ - void GetLocalSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; - - // get device remote queryId from remoteSubscribedMap_ while data change - void GetRemoteSubscribeQueryIds(const std::string &device, std::vector &subscribeQueryIds) const; - // get device remote subscribeQueries from remoteSubscribedMap_ while data change - void GetRemoteSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; - - bool IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const; - - int LocalSubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const; -private: - void ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - int ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - int ActiveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void DeleteSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void RemoveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, - SubscribedTotalMap &subscribedTotalMap); - - void GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, - const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const; - - mutable std::shared_mutex localSubscribeMapLock_; - // subscribe sponsor, key: device, value: pair map - // status 0: active, 1: not active - SubscribeMap localSubscribeMap_; - - // used retry subscribe in db open scene, key: device value: set - std::map> unFinishedLocalAutoSubMap_; - - // subscribe sponsor total query info, key:queryId, value: - // while use_num is 0, delete item from the map - SubscribedTotalMap localSubscribeTotalMap_; - - mutable std::shared_mutex remoteSubscribedMapLock_; - // subscribed, key: device, value: pair map - // status 0: active, 1: not active - SubscribeMap remoteSubscribedMap_; - - // subscribed total query info, key:queryId, value: - // while use_num is 0, delete item from the map - SubscribedTotalMap remoteSubscribedTotalMap_; -}; -} // namespace DistributedDB - -#endif // SINGLE_VER_SUBSCRIBE_MANAGER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp index 0dc5fe0007fddcf822c3fdc0157e10054a83036e..0208d170e9483cc70d93a943ca577c21047320a0 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp @@ -37,8 +37,9 @@ namespace { const int EVENT_INDEX = 1; const int OUTPUT_STATE_INDEX = 2; - // State switch table v1 and v2, has three columns, CurrentState, Event, and OutSate - const std::vector> STATE_SWITCH_TABLE_V2 = { + // drop v1 and v2 table by one optimize, dataSend mode in all version go with slide window mode. + // State switch table v3, has three columns, CurrentState, Event, and OutSate + const std::vector> STATE_SWITCH_TABLE_V3 = { {IDLE, START_SYNC_EVENT, TIME_SYNC}, // In TIME_SYNC state @@ -48,45 +49,25 @@ namespace { // In ABILITY_SYNC state, compare version num and schema {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, - {ABILITY_SYNC, SWITCH_TO_PROCTOL_V1_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, {ABILITY_SYNC, CONTROL_CMD_EVENT, SYNC_CONTROL_CMD}, // In START_INITIACTIVE_DATA_SYNC state, send a sync request, and send first packt of data sync - {START_INITIACTIVE_DATA_SYNC, SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, - {START_INITIACTIVE_DATA_SYNC, RESPONSE_PUSH_REMAINDER_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_INITIACTIVE_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, {START_INITIACTIVE_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {START_INITIACTIVE_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_INITIACTIVE_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_DATA_SYNC}, {START_INITIACTIVE_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, - - // In INACTIVE_PUSH_REMAINDER_DATA state, do initiactive sync, send remainder pcket of data sync - {INACTIVE_PUSH_REMAINDER_DATA, SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, - {INACTIVE_PUSH_REMAINDER_DATA, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, - {INACTIVE_PUSH_REMAINDER_DATA, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {INACTIVE_PUSH_REMAINDER_DATA, INNER_ERR_EVENT, INNER_ERR}, - {INACTIVE_PUSH_REMAINDER_DATA, RE_SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, - {INACTIVE_PUSH_REMAINDER_DATA, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + {START_INITIACTIVE_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_DATA_SYNC}, // In START_PASSIVE_DATA_SYNC state, do response pull request, and send first packt of data sync - {START_PASSIVE_DATA_SYNC, SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_PASSIVE_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, {START_PASSIVE_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, {START_PASSIVE_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {START_PASSIVE_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_PASSIVE_DATA_SYNC, RE_SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_PASSIVE_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - - // In PASSIVE_PUSH_REMAINDER_DATA state, do passive sync, send remainder pcket of data sync - {PASSIVE_PUSH_REMAINDER_DATA, SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, - {PASSIVE_PUSH_REMAINDER_DATA, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, - {PASSIVE_PUSH_REMAINDER_DATA, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {PASSIVE_PUSH_REMAINDER_DATA, INNER_ERR_EVENT, INNER_ERR}, - {PASSIVE_PUSH_REMAINDER_DATA, RE_SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, - {PASSIVE_PUSH_REMAINDER_DATA, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + {START_PASSIVE_DATA_SYNC, RE_SEND_DATA_EVENT, START_PASSIVE_DATA_SYNC}, // In WAIT_FOR_RECEIVE_DATA_FINISH, {WAIT_FOR_RECEIVE_DATA_FINISH, RECV_FINISHED_EVENT, SYNC_TASK_FINISHED}, @@ -108,58 +89,6 @@ namespace { {SYNC_TIME_OUT, ANY_EVENT, SYNC_TASK_FINISHED}, {INNER_ERR, ANY_EVENT, SYNC_TASK_FINISHED}, }; - - // State switch table v3, has three columns, CurrentState, Event, and OutSate - const std::vector> STATE_SWITCH_TABLE_V3 = { - {IDLE, START_SYNC_EVENT, TIME_SYNC}, - - // In TIME_SYNC state - {TIME_SYNC, TIME_SYNC_FINISHED_EVENT, ABILITY_SYNC}, - {TIME_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {TIME_SYNC, INNER_ERR_EVENT, INNER_ERR}, - - // In ABILITY_SYNC state, compare version num and schema - {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, - {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, - {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {ABILITY_SYNC, CONTROL_CMD_EVENT, SYNC_CONTROL_CMD}, - - // In START_INITIACTIVE_SLIDING_DATA_SYNC state, send a sync request, and send first packt of data sync - {START_INITIACTIVE_SLIDING_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - {START_INITIACTIVE_SLIDING_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, - - // In START_PASSIVE_SLIDING_DATA_SYNC state, do response pull request, and send first packt of data sync - {START_PASSIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - {START_PASSIVE_SLIDING_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, - {START_PASSIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {START_PASSIVE_SLIDING_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, - {START_PASSIVE_SLIDING_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - {START_PASSIVE_SLIDING_DATA_SYNC, RE_SEND_DATA_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - - // In WAIT_FOR_RECEIVE_DATA_FINISH, - {WAIT_FOR_RECEIVE_DATA_FINISH, RECV_FINISHED_EVENT, SYNC_TASK_FINISHED}, - {WAIT_FOR_RECEIVE_DATA_FINISH, START_PULL_RESPONSE_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, - {WAIT_FOR_RECEIVE_DATA_FINISH, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {WAIT_FOR_RECEIVE_DATA_FINISH, INNER_ERR_EVENT, INNER_ERR}, - {WAIT_FOR_RECEIVE_DATA_FINISH, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - - {SYNC_CONTROL_CMD, SEND_FINISHED_EVENT, SYNC_TASK_FINISHED}, - {SYNC_CONTROL_CMD, TIME_OUT_EVENT, SYNC_TIME_OUT}, - {SYNC_CONTROL_CMD, INNER_ERR_EVENT, INNER_ERR}, - {SYNC_CONTROL_CMD, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - - // In SYNC_TASK_FINISHED, - {SYNC_TASK_FINISHED, ALL_TASK_FINISHED_EVENT, IDLE}, - {SYNC_TASK_FINISHED, START_SYNC_EVENT, TIME_SYNC}, - - // SYNC_TIME_OUT and INNE_ERR state, just do some exception resolve - {SYNC_TIME_OUT, ANY_EVENT, SYNC_TASK_FINISHED}, - {INNER_ERR, ANY_EVENT, SYNC_TASK_FINISHED}, - }; } std::mutex SingleVerSyncStateMachine::stateSwitchTableLock_; @@ -197,7 +126,6 @@ int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, ISyncInterf timeSync_ = std::make_unique(); dataSync_ = std::make_shared(); abilitySync_ = std::make_unique(); - dataSyncWithSlidingWindow_ = std::make_unique(); errCode = timeSync_->Initialize(communicator, metaData, syncInterface, context->GetDeviceId()); if (errCode != E_OK) { @@ -215,14 +143,7 @@ int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, ISyncInterf currentState_ = IDLE; context_ = static_cast(context); syncInterface_ = static_cast(syncInterface); - errCode = dataSyncWithSlidingWindow_->ReceiverInit(context_, dataSync_); - if (errCode != E_OK) { - goto ERROR_OUT; - } - errCode = dataSyncWithSlidingWindow_->SenderInit(context_, dataSync_); - if (errCode != E_OK) { - goto ERROR_OUT; - } + InitStateSwitchTables(); InitStateMapping(); return E_OK; @@ -324,11 +245,10 @@ void SingleVerSyncStateMachine::AbortInner() { LOGE("[StateMachine][AbortInner] error occurred,abort,label=%s,dev=%s", dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - int mode = SyncOperation::TransferSyncMode(context_->GetMode()); - if (mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL || context_->IsKilled()) { - dataSyncWithSlidingWindow_->ReceiverClear(); + if (context_->IsKilled()) { + dataSync_->ClearDataMsg(); } - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); ContinueToken token; context_->GetContinueToken(token); if (token != nullptr) { @@ -384,7 +304,6 @@ void SingleVerSyncStateMachine::InitStateSwitchTables() return; } - InitStateSwitchTable(SINGLE_VER_SYNC_PROCTOL_V2, STATE_SWITCH_TABLE_V2); InitStateSwitchTable(SINGLE_VER_SYNC_PROCTOL_V3, STATE_SWITCH_TABLE_V3); std::sort(stateSwitchTables_.begin(), stateSwitchTables_.end(), [](const auto &tableA, const auto &tableB) { @@ -419,54 +338,17 @@ void SingleVerSyncStateMachine::InitStateMapping() { stateMapping_[TIME_SYNC] = std::bind(&SingleVerSyncStateMachine::DoTimeSync, this); stateMapping_[ABILITY_SYNC] = std::bind(&SingleVerSyncStateMachine::DoAbilitySync, this); - stateMapping_[START_INITIACTIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveDataSync, - this); - stateMapping_[START_PASSIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoPassiveDataSync, this); - stateMapping_[INACTIVE_PUSH_REMAINDER_DATA] = std::bind(&SingleVerSyncStateMachine::DoInitiactivePushRemainderData, - this); - stateMapping_[PASSIVE_PUSH_REMAINDER_DATA] = std::bind(&SingleVerSyncStateMachine::DoPassivePushRemainderData, - this); stateMapping_[WAIT_FOR_RECEIVE_DATA_FINISH] = std::bind(&SingleVerSyncStateMachine::DoWaitForDataRecv, this); stateMapping_[SYNC_TASK_FINISHED] = std::bind(&SingleVerSyncStateMachine::DoSyncTaskFinished, this); stateMapping_[SYNC_TIME_OUT] = std::bind(&SingleVerSyncStateMachine::DoTimeout, this); stateMapping_[INNER_ERR] = std::bind(&SingleVerSyncStateMachine::DoInnerErr, this); - stateMapping_[START_INITIACTIVE_SLIDING_DATA_SYNC] = + stateMapping_[START_INITIACTIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow, this); - stateMapping_[START_PASSIVE_SLIDING_DATA_SYNC] = + stateMapping_[START_PASSIVE_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow, this); stateMapping_[SYNC_CONTROL_CMD] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveControlSync, this); } -Event SingleVerSyncStateMachine::DoInitiactiveDataSync() -{ - int errCode = E_OK; - int mode = SyncOperation::TransferSyncMode(context_->GetMode()); - switch (mode) { - case SyncModeType::PUSH: - context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); - errCode = dataSync_->PushStart(context_); - break; - case SyncModeType::PULL: - context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - errCode = dataSync_->PullRequestStart(context_); - break; - case SyncModeType::PUSH_AND_PULL: - errCode = dataSync_->PushPullStart(context_); - break; - case SyncModeType::RESPONSE_PULL: - // In response pull mode, reminader data should send in - // PASSIVE_PUSH_REMAINDER_DATA - return Event::RESPONSE_PUSH_REMAINDER_EVENT; - default: - errCode = -E_NOT_SUPPORT; - break; - } - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() { LOGD("[StateMachine][activeDataSync] mode=%d,label=%s,dev=%s", context_->GetMode(), @@ -476,19 +358,19 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() case SyncModeType::PUSH: case SyncModeType::QUERY_PUSH: context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; case SyncModeType::PULL: case SyncModeType::QUERY_PULL: context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; case SyncModeType::PUSH_AND_PULL: case SyncModeType::QUERY_PUSH_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; case SyncModeType::RESPONSE_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); + errCode = dataSync_->SyncStart(context_->GetMode(), context_); break; default: errCode = -E_NOT_SUPPORT; @@ -509,23 +391,6 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() return (ignoreInnerErr && event == INNER_ERR_EVENT) ? SEND_FINISHED_EVENT : event; } -Event SingleVerSyncStateMachine::DoPassiveDataSync() -{ - { - RefObject::AutoLock lock(context_); - if (context_->GetRspTargetQueueSize() != 0) { - PreStartPullResponse(); - } else { - return RESPONSE_TASK_FINISHED_EVENT; - } - } - int errCode = dataSync_->PullResponseStart(context_); - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - Event SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow() { { @@ -536,7 +401,7 @@ Event SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow() return RESPONSE_TASK_FINISHED_EVENT; } } - int errCode = dataSyncWithSlidingWindow_->SenderStart(SyncModeType::RESPONSE_PULL, context_, dataSync_); + int errCode = dataSync_->SyncStart(SyncModeType::RESPONSE_PULL, context_); if (errCode != E_OK) { LOGW("[SingleVerSyncStateMachine][DoPassiveDataSyncWithSlidingWindow] response pull send failed[%d]", errCode); return RESPONSE_TASK_FINISHED_EVENT; @@ -569,39 +434,6 @@ int SingleVerSyncStateMachine::HandleControlAckRecv(const Message *inMsg) return E_OK; } -Event SingleVerSyncStateMachine::DoInitiactivePushRemainderData() -{ - int errCode; - int mode = SyncOperation::TransferSyncMode(context_->GetMode()); - switch (mode) { - case SyncModeType::PULL: - case SyncModeType::RESPONSE_PULL: - // In pull or response pul mode, don't need to do INACTIVE_PUSH - return Event::SEND_FINISHED_EVENT; - case SyncModeType::PUSH: - case SyncModeType::PUSH_AND_PULL: - errCode = dataSync_->PushStart(context_); - break; - default: - errCode = -E_INTERNAL_ERROR; - break; - } - - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - -Event SingleVerSyncStateMachine::DoPassivePushRemainderData() -{ - int errCode = dataSync_->PullResponseStart(context_); - if (errCode == E_OK) { - return Event::WAIT_ACK_EVENT; - } - return TransformErrCodeToEvent(errCode); -} - Event SingleVerSyncStateMachine::DoWaitForDataRecv() const { if (context_->GetRspTargetQueueSize() != 0) { @@ -676,17 +508,13 @@ Event SingleVerSyncStateMachine::GetEventAfterTimeSync(int mode, bool isEarliest if (mode == SyncModeType::SUBSCRIBE_QUERY || mode == SyncModeType::UNSUBSCRIBE_QUERY) { return Event::CONTROL_CMD_EVENT; } - if (isEarliestVersion) { - LOGI("remote version is 0, switch to v1 proctol"); - return Event::SWITCH_TO_PROCTOL_V1_EVENT; - } return Event::ABILITY_SYNC_FINISHED_EVENT; } Event SingleVerSyncStateMachine::DoSyncTaskFinished() { StopWatchDog(); - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); RefObject::AutoLock lock(syncContext_); int errCode = ExecNextTask(); if (errCode == E_OK) { @@ -786,6 +614,16 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) { + TimeOffset offset = 0; + uint32_t timeout = TIME_SYNC_WAIT_TIME; + timeout = communicator_->GetTimeout(context_->GetDeviceId()); + // If message is data sync request, we should check timeoffset. + int errCode = timeSync_->GetTimeOffset(offset, timeout); + if (errCode != E_OK) { + LOGE("[StateMachine][HandleDataRequestRecv] GetTimeOffset err! errCode=%d", errCode); + return errCode; + } + context_->SetTimeOffset(offset); PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); if (performance != nullptr) { performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); @@ -802,7 +640,7 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) // So we need to send save data notify to keep remote alive. bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); WaterMark pullEndWaterkark = 0; - int errCode = dataSync_->DataRequestRecv(context_, inMsg, pullEndWaterkark); + errCode = dataSync_->DataRequestRecv(context_, inMsg, pullEndWaterkark); if (performance != nullptr) { performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); } @@ -823,22 +661,14 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) return E_OK; } -int SingleVerSyncStateMachine::PreHandleAckRecv(const Message *inMsg) -{ - if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - return dataSyncWithSlidingWindow_->PreHandleSenderAckRecv(inMsg); - } - return E_OK; -} - void SingleVerSyncStateMachine::HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg, bool ignoreInnerErr) { if (errCode == -E_RE_SEND_DATA) { // LOCAL_WATER_MARK_NOT_INIT - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); } if (errCode == -E_NO_DATA_SEND || errCode == -E_SEND_DATA) { - int ret = dataSyncWithSlidingWindow_->SenderAckRecv(inMsg); + int ret = dataSync_->TryContinueSync(context_, inMsg); if (ret == -E_FINISHED) { SwitchStateAndStep(Event::SEND_FINISHED_EVENT); return; @@ -862,7 +692,7 @@ void SingleVerSyncStateMachine::NeedAbilitySyncHandle() currentRemoteVersionId_ = context_->GetRemoteSoftwareVersionId(); } abilitySync_->SetAbilitySyncFinishedStatus(false); - dataSyncWithSlidingWindow_->SenderClear(); + dataSync_->ClearSyncStatus(); } int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) @@ -881,22 +711,16 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) if (IsNeedResetWatchdog(inMsg)) { (void)ResetWatchDog(); } - int errCode = PreHandleAckRecv(inMsg); - if (errCode != E_OK) { + if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && !dataSync_->AckPacketIdCheck(inMsg)) { // packetId not match but sequence id matched scene, means resend map has be rebuilt // this is old ack, shoulb be dropped and wait for the same packetId sequence. - if (errCode == -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG) { - return E_OK; - } - // this means error happened,should stop the sync task. - SwitchStateAndStep(TransformErrCodeToEvent(errCode)); - return errCode; + return E_OK; } // AckRecv will save meta data, it may cost a long time. if another thread is saving data // So we need to send save data notify to keep remote alive. // eg. remote do pull sync bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); - errCode = dataSync_->AckRecv(context_, inMsg); + int errCode = dataSync_->AckRecv(context_, inMsg); if (isNeedStop) { StopSaveDataNotify(); } @@ -909,10 +733,6 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) bool ignoreInnerErr = inMsg->GetSessionId() == context_->GetResponseSessionId() && context_->GetRequestSessionId() != 0; DataAckRecvErrCodeHandle(errCode, !ignoreInnerErr); - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - ResponsePullError(errCode, ignoreInnerErr); - return errCode; - } HandleDataAckRecvWithSlidingWindow(errCode, inMsg, ignoreInnerErr); return errCode; } @@ -920,26 +740,11 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) int SingleVerSyncStateMachine::DataPktRecv(Message *inMsg) { PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - int errCode; - TimeOffset offset = 0; - uint32_t timeout = TIME_SYNC_WAIT_TIME; - + int errCode = E_OK; switch (inMsg->GetMessageType()) { case TYPE_REQUEST: - timeout = communicator_->GetTimeout(context_->GetDeviceId()); - // If message is data sync request, we should check timeoffset. - errCode = timeSync_->GetTimeOffset(offset, timeout); - if (errCode != E_OK) { - LOGE("[StateMachine][DataPktRecv] GetTimeOffset err! errCode=%d", errCode); - return errCode; - } - context_->SetTimeOffset(offset); - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - // higher than 102 version may go to here, but it is ok not use slwr,after abilitysync would go slwr. - errCode = HandleDataRequestRecv(inMsg); - } else { - errCode = dataSyncWithSlidingWindow_->Receive(inMsg); - } + ScheduleMsgAndHandle(inMsg); + errCode = -E_NOT_NEED_DELETE_MSG; break; case TYPE_RESPONSE: case TYPE_NOTIFY: @@ -956,6 +761,43 @@ int SingleVerSyncStateMachine::DataPktRecv(Message *inMsg) return errCode; } +void SingleVerSyncStateMachine::ScheduleMsgAndHandle(Message *inMsg) +{ + dataSync_->PutDataMsg(inMsg); + while (true) { + bool isNeedHandle = true; + bool isNeedContinue = true; + Message *msg = dataSync_->MoveNextDataMsg(context_, isNeedHandle, isNeedContinue); + if (!isNeedContinue) { + break; + } + if (msg == nullptr) { + if (dataSync_->IsNeedReloadQueue()) { + continue; + } + break; + } + bool isNeedClearMap = false; + if (isNeedHandle) { + int errCode = HandleDataRequestRecv(msg); + if (context_->IsReceiveWaterMarkErr() || errCode == -E_NEED_ABILITY_SYNC) { + isNeedClearMap = true; + } + if (errCode == -E_TIMEOUT) { + isNeedHandle = false; + } + } else { + dataSync_->SendFinishedDataAck(context_, msg); + } + if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { + // for lower version, no need to handle map schedule info, just reset schedule working status + isNeedHandle = false; + } + dataSync_->ScheduleInfoHandle(isNeedHandle, isNeedClearMap, msg); + delete msg; + } +} + int SingleVerSyncStateMachine::ControlPktRecv(Message *inMsg) { int errCode = E_OK; @@ -1034,7 +876,6 @@ int SingleVerSyncStateMachine::TimeMarkSyncRecv(const Message *inMsg) void SingleVerSyncStateMachine::Clear() { - dataSyncWithSlidingWindow_ = nullptr; dataSync_ = nullptr; timeSync_ = nullptr; abilitySync_ = nullptr; @@ -1073,18 +914,7 @@ bool SingleVerSyncStateMachine::IsPacketValid(const Message *inMsg) const inMsg->GetMessageId() == CONTROL_SYNC_MESSAGE) { return true; } - - // sliding window don't need to check sequenceId here, just return - if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - return true; - } - - if (isResponseType && (inMsg->GetSequenceId() != context_->GetSequenceId())) { - LOGE("[StateMachine][IsPacketValid] Message is invalid,inMsg SequenceId=%d,seqId=%d,syncId=%d", - inMsg->GetSequenceId(), context_->GetSequenceId(), context_->GetSyncId()); - return false; - } - + // sequenceId will be checked in dataSync return true; } @@ -1100,23 +930,6 @@ void SingleVerSyncStateMachine::PreStartPullResponse() context_->SetQuery(target.GetQuery()); } -bool SingleVerSyncStateMachine::IsRightDataResponsePkt(const Message *inMsg) const -{ - if (inMsg->GetMessageId() != DATA_SYNC_MESSAGE || inMsg->GetMessageId() != QUERY_SYNC_MESSAGE) { - return false; - } - - if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - return false; - } - - if ((context_->GetMode() != SyncModeType::INVALID_MODE) && (inMsg->GetMessageType() == TYPE_RESPONSE)) { - return true; - } - - return false; -} - bool SingleVerSyncStateMachine::CheckIsStartPullResponse() const { // Other state will step to do pull response, only this statem we need to step the statemachine @@ -1136,10 +949,6 @@ int SingleVerSyncStateMachine::MessageCallbackPre(const Message *inMsg) if (!IsPacketValid(inMsg)) { return -E_INVALID_ARGS; } - - if (IsRightDataResponsePkt(inMsg)) { - context_->IncSequenceId(); - } return E_OK; } @@ -1238,11 +1047,6 @@ Event SingleVerSyncStateMachine::TransforTimeOutErrCodeToEvent() } } -void SingleVerSyncStateMachine::SetSlidingWindowSenderErr(bool isErr) -{ - dataSyncWithSlidingWindow_->SetSenderErr(isErr); -} - bool SingleVerSyncStateMachine::IsNeedErrCodeHandle(uint32_t sessionId) const { // omit to set sessionId so version_3 should skip to compare sessionid. @@ -1320,12 +1124,6 @@ void SingleVerSyncStateMachine::DataAckRecvErrCodeHandle(int errCode, bool handl case -E_NEED_ABILITY_SYNC: NeedAbilitySyncHandle(); break; - case -E_NO_DATA_SEND: - // when version is higher than 102, this step will be handle in slide windows function. - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0 && handleError) { - context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - } - break; case -E_NOT_PERMIT: if (handleError) { context_->SetOperationStatus(SyncOperation::OP_PERMISSION_CHECK_FAILED); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h index 5fbb258f12703e76440453fed7007a0254b6622d..965c13eebaf8fec07b65ec7b4f4d6f279cdd8b8b 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h @@ -24,7 +24,6 @@ #include "meta_data.h" #include "semaphore_utils.h" #include "single_ver_data_sync.h" -#include "single_ver_data_sync_with_sliding_window.h" #include "single_ver_sync_task_context.h" #include "sync_state_machine.h" #include "sync_target.h" @@ -38,16 +37,12 @@ namespace { IDLE = 0, TIME_SYNC, ABILITY_SYNC, - START_INITIACTIVE_DATA_SYNC, // used to do sync started by local device - START_PASSIVE_DATA_SYNC, // used to do sync response remote device - INACTIVE_PUSH_REMAINDER_DATA, // push remainded data if initactive sync data more than on frame - PASSIVE_PUSH_REMAINDER_DATA, // push remainded data if passive sync data more than on frame WAIT_FOR_RECEIVE_DATA_FINISH, // all data send finished, wait for data revice if has pull request SYNC_TASK_FINISHED, // current sync task finihsed, try to schedule next sync task SYNC_TIME_OUT, INNER_ERR, - START_INITIACTIVE_SLIDING_DATA_SYNC, // used to do sync started by local device, use sliding window - START_PASSIVE_SLIDING_DATA_SYNC, // used to do pull response, use sliding window + START_INITIACTIVE_DATA_SYNC, // used to do sync started by local device, use sliding window + START_PASSIVE_DATA_SYNC, // used to do pull response, use sliding window SYNC_CONTROL_CMD // used to send control cmd. }; @@ -56,12 +51,10 @@ namespace { TIME_SYNC_FINISHED_EVENT, ABILITY_SYNC_FINISHED_EVENT, VERSION_NOT_SUPPOR_EVENT, - SWITCH_TO_PROCTOL_V1_EVENT, SEND_DATA_EVENT, SEND_FINISHED_EVENT, RECV_FINISHED_EVENT, NEED_ABILITY_SYNC_EVENT, - RESPONSE_PUSH_REMAINDER_EVENT, RESPONSE_TASK_FINISHED_EVENT, START_PULL_RESPONSE_EVENT, WAIT_ACK_EVENT, @@ -94,8 +87,6 @@ public: int HandleDataRequestRecv(const Message *inMsg); - void SetSlidingWindowSenderErr(bool isErr); - bool IsNeedErrCodeHandle(uint32_t sessionId) const; void PushPullDataRequestEvokeErrHandle(); @@ -151,18 +142,6 @@ private: // Do AbilitySync, for first sync Event DoAbilitySync(); - // Do data packet sync for initiactive sync operation, if need pull, there need send a pull request. - Event DoInitiactiveDataSync(); - - // Do data packet sync for response pull request. - Event DoPassiveDataSync(); - - // Push remainder data to remote - Event DoInitiactivePushRemainderData(); - - // Push remainder data to remote for response pull request - Event DoPassivePushRemainderData(); - // Waiting for pull data revice finish, if coming a pull request, should goto START_PASSIVE_DATA_SYNC state Event DoWaitForDataRecv() const; @@ -193,14 +172,14 @@ private: int DataPktRecv(Message *inMsg); + void ScheduleMsgAndHandle(Message *inMsg); + int ControlPktRecv(Message *inMsg); void NeedAbilitySyncHandle(); int HandleDataAckRecv(const Message *inMsg); - int PreHandleAckRecv(const Message *inMsg); - void HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg, bool ignoreInnerErr); void ResponsePullError(int errCode, bool ignoreInnerErr); @@ -211,8 +190,6 @@ private: void PreStartPullResponse(); - bool IsRightDataResponsePkt(const Message *inMsg) const; - bool CheckIsStartPullResponse() const; int MessageCallbackPre(const Message *inMsg); @@ -245,7 +222,6 @@ private: std::unique_ptr timeSync_; std::unique_ptr abilitySync_; std::shared_ptr dataSync_; - std::unique_ptr dataSyncWithSlidingWindow_; uint64_t currentRemoteVersionId_; std::map stateMapping_; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp index d07b32898fb788b571d5e6b9138141dab3f6e8db..cbc1fbb563433f716f8da2d65d8399378f7163f3 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp @@ -307,27 +307,6 @@ void SingleVerSyncTaskContext::StopFeedDogForSync(SyncDirectionFlag flag) stateMachine_->StopFeedDogForSync(flag); } -void SingleVerSyncTaskContext::SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange) -{ - dataTimeRange_ = dataTimeRange; -} - -SyncTimeRange SingleVerSyncTaskContext::GetDataTimeRange() const -{ - return dataTimeRange_; -} - - -void SingleVerSyncTaskContext::SetSessionEndTimeStamp(TimeStamp end) -{ - sessionEndTimeStamp_ = end; -} - -TimeStamp SingleVerSyncTaskContext::GetSessionEndTimeStamp() const -{ - return sessionEndTimeStamp_; -} - int SingleVerSyncTaskContext::HandleDataRequestRecv(const Message *msg) { return static_cast(stateMachine_)->HandleDataRequestRecv(msg); @@ -343,11 +322,6 @@ void SingleVerSyncTaskContext::SetReceiveWaterMarkErr(bool isErr) isReceiveWaterMarkErr_ = isErr; } -void SingleVerSyncTaskContext::SetSlidingWindowSenderErr(bool isErr) -{ - static_cast(stateMachine_)->SetSlidingWindowSenderErr(isErr); -} - void SingleVerSyncTaskContext::SetRemoteSeccurityOption(SecurityOption secOption) { remoteSecOption_ = secOption; @@ -546,7 +520,7 @@ bool SingleVerSyncTaskContext::IsCurrentSyncTaskCanBeSkipped() const return false; } if (localWaterMark > maxTimeStampInDb) { - LOGD("skip current push task, deviceId_ = %s, localWaterMark = %llu, maxTimeStampInDb = %llu", + LOGI("skip current push task, deviceId_ = %s, localWaterMark = %llu, maxTimeStampInDb = %llu", STR_MASK(deviceId_), localWaterMark, maxTimeStampInDb); return true; } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h index 221d65e9953ccf3a0c202230d1307eaf058bae12..85f29c2123aae99e81687629ddc2190f813b471d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h @@ -88,28 +88,14 @@ public: // stop timer to ResetWatchDog when sync data one (key,value) size bigger than mtu void StopFeedDogForSync(SyncDirectionFlag flag); - SyncTimeRange GetDataTimeRange() const; - virtual int HandleDataRequestRecv(const Message *msg); - // if sended by sliding window, set the start and and timeStamp of data in a sequence - void SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange); - - // if sended by sliding window, set the last data timeStamp in a sync session - void SetSessionEndTimeStamp(TimeStamp end); - - // if sended by sliding window, get the last data timeStamp in a sync session - TimeStamp GetSessionEndTimeStamp() const; - // is receive warterMark err bool IsReceiveWaterMarkErr() const; // set receive warterMark err void SetReceiveWaterMarkErr(bool isErr); - // set sliding window sender err - void SetSlidingWindowSenderErr(bool isErr); - void SetRemoteSeccurityOption(SecurityOption secOption); SecurityOption GetRemoteSeccurityOption() const; @@ -173,11 +159,6 @@ private: SyncStrategy syncStrategy_; bool isSchemaSync_ = false; - // normal data or delete data start timestamp and end timestamp,recorded for slws resend. - SyncTimeRange dataTimeRange_; - // in a sync session, the last data timeStamp - TimeStamp sessionEndTimeStamp_ = 0; - // is receive waterMark err, peerWaterMark bigger than remote localWaterMark bool isReceiveWaterMarkErr_ = false; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp deleted file mode 100755 index 3cb5c8a725d059d6120fcbdd68882b2c1e1b3679..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sliding_window_receiver.h" -#include "sync_task_context.h" -#include "db_common.h" - -namespace DistributedDB { -SlidingWindowReceiver::~SlidingWindowReceiver() -{ - Clear(); - dataSync_ = nullptr; - context_ = nullptr; -} - -void SlidingWindowReceiver::Clear() -{ - StopTimer(); - std::lock_guard lock(lock_); - ClearMap(); - isWaterMarkErrHappened_ = false; -} - -int SlidingWindowReceiver::Initialize(SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - if (context == nullptr || dataSync == nullptr) { - LOGE("[slwr] Initialize invalid args"); - return -E_INVALID_ARGS; - } - context_ = context; - dataSync_ = dataSync; - return E_OK; -} - -int SlidingWindowReceiver::Receive(Message *inMsg) -{ - LOGD("[slwr] receive msg"); - if (inMsg == nullptr) { - return -E_INVALID_ARGS; - } - int errCode = PutMsg(inMsg); - if (errCode == E_OK) { - StopTimer(); - StartTimer(); - DealMsg(); - return -E_NOT_NEED_DELETE_MSG; - } - return errCode; -} - -int SlidingWindowReceiver::PutMsg(Message *inMsg) -{ - const DataRequestPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - uint32_t sessionId = inMsg->GetSessionId(); - uint32_t sequenceId = inMsg->GetSequenceId(); - bool isLastSequence = packet->IsLastSequence(); - std::unique_lock lock(lock_); - if (workingId_ != 0 && sessionId_ != 0) { - LOGI("[PutMsg] task is running, wait for workdingId=%u end,seId=%u", workingId_, sequenceId); - workingTaskcv_.wait(lock, [this] { return workingId_ == 0; }); - } - if (sessionId_ != sessionId) { - ResetInfo(); - sessionId_ = sessionId; - messageMap_[sequenceId] = inMsg; - SetEndField(isLastSequence, sequenceId); - return E_OK; - } - // maybe remote has not receive ack, we resend ack. - if (sequenceId <= hasFinishedMaxId_) { - LOGI("[slwr] seId=%u,FinishedMId_=%u,label=%s", sequenceId, hasFinishedMaxId_, dataSync_->GetLabel().c_str()); - lock.unlock(); - dataSync_->SendDataAck(context_, inMsg, E_OK, 0); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - int errCode = ErrHandle(sequenceId, packet->GetPacketId()); - if (errCode != E_OK) { - return errCode; - } - if (messageMap_.count(sequenceId) > 0) { - LOGI("[slwr] PutMsg sequenceId already in map label=%s,deviceId=%s", - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - // it cannot be null here - const auto *cachePacket = messageMap_[sequenceId]->GetObject(); - if (cachePacket->GetPacketId() > packet->GetPacketId()) { - LOGI("[slwr] PutMsg receive old packet %u, new packet is %u", - packet->GetPacketId(), cachePacket->GetPacketId()); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - delete messageMap_[sequenceId]; - messageMap_[sequenceId] = nullptr; - } - messageMap_[sequenceId] = inMsg; - SetEndField(isLastSequence, sequenceId); - return E_OK; -} - -void SlidingWindowReceiver::DealMsg() -{ - while (true) { - Message *msg = nullptr; - uint64_t curPacketId = 0; - { - std::lock_guard lock(lock_); - if (workingId_ != 0 || messageMap_.count(hasFinishedMaxId_ + 1) == 0) { - LOGI("[slwr] DealMsg do nothing workingId_=%u,hasFinishedMaxId_=%u,label=%s,deviceId=%s", - workingId_, hasFinishedMaxId_, dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - return; - } - uint32_t curWorkingId = hasFinishedMaxId_ + 1; - msg = messageMap_[curWorkingId]; - messageMap_.erase(curWorkingId); - const auto *packet = msg->GetObject(); - curPacketId = packet->GetPacketId(); - if (curPacketId != 0 && curPacketId < hasFinishedPacketId_) { - delete msg; - msg = nullptr; - LOGI("[slwr] DealMsg ignore msg, packetId:%u,label=%s,deviceId=%s", - curPacketId, dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - continue; - } - workingId_ = curWorkingId; - LOGI("[slwr] DealMsg workingId_=%u,label=%s,deviceId=%s", workingId_, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - } - int errCode = context_->HandleDataRequestRecv(msg); - delete msg; - msg = nullptr; - { - std::lock_guard lock(lock_); - workingId_ = 0; - bool isWaterMarkErr = context_->IsReceiveWaterMarkErr(); - if (isWaterMarkErr || errCode == -E_NEED_ABILITY_SYNC) { - ClearMap(); - hasFinishedMaxId_ = 0; - endId_ = 0; - isWaterMarkErrHappened_ = true; - } else { - hasFinishedMaxId_++; - hasFinishedPacketId_ = curPacketId; - LOGI("[slwr] DealMsg ok hasFinishedMaxId_=%u,label=%s,deviceId=%s", hasFinishedMaxId_, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - } - context_->SetReceiveWaterMarkErr(false); - workingTaskcv_.notify_all(); - } - } -} - -int SlidingWindowReceiver::TimeOut(TimerId timerId) -{ - { - std::lock_guard lock(lock_); - LOGI("[slwr] TimeOut,timerId[%llu], timerId_[%llu]", timerId, timerId_); - if (timerId == timerId_) { - ClearMap(); - } - timerId_ = 0; - } - RuntimeContext::GetInstance()->RemoveTimer(timerId); - return E_OK; -} - -void SlidingWindowReceiver::StartTimer() -{ - std::lock_guard lock(lock_); - TimerId timerId = 0; - RefObject::IncObjRef(context_); - TimerAction timeOutCallback = std::bind(&SlidingWindowReceiver::TimeOut, this, std::placeholders::_1); - int errCode = RuntimeContext::GetInstance()->SetTimer(IDLE_TIME_OUT, timeOutCallback, - [this]() { - int ret = RuntimeContext::GetInstance()->ScheduleTask([this]() { - RefObject::DecObjRef(context_); - }); - if (ret != E_OK) { - LOGE("[slwr] timer finalizer ScheduleTask, errCode=%d", ret); - } - }, timerId); - if (errCode != E_OK) { - RefObject::DecObjRef(context_); - LOGE("[slwr] timer ScheduleTask, errCode=%d", errCode); - return; - } - StopTimer(timerId_); - timerId_ = timerId; - LOGD("[slwr] StartTimer timerId[%llu] success", timerId_); -} - -void SlidingWindowReceiver::StopTimer() -{ - TimerId timerId; - { - std::lock_guard lock(lock_); - timerId = timerId_; - if (timerId_ == 0) { - return; - } - timerId_ = 0; - } - StopTimer(timerId); -} - -void SlidingWindowReceiver::StopTimer(TimerId timerId) -{ - if (timerId == 0) return; - LOGD("[slwr] StopTimer,remove Timer id[%llu]", timerId); - RuntimeContext::GetInstance()->RemoveTimer(timerId); -} - -void SlidingWindowReceiver::ClearMap() -{ - LOGD("[slwr] ClearMap"); - for (auto &iter : messageMap_) { - delete iter.second; - iter.second = nullptr; - } - messageMap_.clear(); -} - -void SlidingWindowReceiver::ResetInfo() -{ - ClearMap(); - hasFinishedMaxId_ = 0; - hasFinishedPacketId_ = 0; - workingId_ = 0; - endId_ = 0; - isWaterMarkErrHappened_ = false; -} - -int SlidingWindowReceiver::ErrHandle(uint32_t sequenceId, uint64_t packetId) -{ - if (sequenceId == workingId_ || (endId_ != 0 && sequenceId > endId_) - || (packetId > 0 && packetId < hasFinishedPacketId_)) { - LOGI("[slwr] PutMsg sequenceId:%u, endId_:%u, workingId_:%u, packetId:%u!", - sequenceId, endId_, workingId_, packetId); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - // if waterMark err when DealMsg(), the waterMark of msg in messageMap_ is also err, so we clear messageMap_ - // but before the sender receive the waterMark err ack, the receiver may receive waterMark err msg, so before - // receiver receive the sequenceId 1 msg, it will drop msg. - // note, there is a low probability risk, if sender receive the waterMark err ack, it correct the warterMark then - // resend data, if the msg of bigger sequenceId arrive earlier than sequenceId 1, it will be drop, the sender - // will sync timeout. - if (isWaterMarkErrHappened_) { - if (sequenceId == 1) { - isWaterMarkErrHappened_ = false; - } else { - LOGI("[slwr] PutMsg With waterMark Error sequenceId:%u, endId_:%u, workingId_:%u, packetId:%u!", - sequenceId, endId_, workingId_, packetId); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; // drop invalid packet - } - } - return E_OK; -} - -void SlidingWindowReceiver::SetEndField(bool isLastSequence, uint32_t sequenceId) -{ - if (isLastSequence) { - endId_ = sequenceId; - } -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp deleted file mode 100755 index 32df1ca7619976d9d156fa71191e36383496193e..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "sliding_window_sender.h" -#include "db_common.h" -namespace DistributedDB { -SlidingWindowSender::~SlidingWindowSender() -{ - Clear(); - context_ = nullptr; - dataSync_ = nullptr; -} - -int SlidingWindowSender::ParamCheck(int32_t mode, const SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - if (context == nullptr) { - LOGE("[slws] SendStart invalid context"); - return -E_INVALID_ARGS; - } - if (dataSync == nullptr) { - LOGE("[slws] SendStart invalid context"); - return -E_INVALID_ARGS; - } - return E_OK; -} - -void SlidingWindowSender::Init(int32_t inMode, SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - isErr_ = false; - mode_ = inMode; - context_->SetSessionEndTimeStamp(0); - context_->ReSetSequenceId(); - maxSequenceIdhasSent_ = 0; - windowSize_ = MAX_WINDOW_SIZE; - int mode = SyncOperation::TransferSyncMode(inMode); - if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL) { - sessionId_ = context->GetRequestSessionId(); - } else { - sessionId_ = context->GetResponseSessionId(); - } -} - -int SlidingWindowSender::Initialize(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync) -{ - if (context == nullptr || dataSync == nullptr) { - LOGE("[slws] Initialize invalid args"); - return -E_INVALID_ARGS; - } - context_ = context; - dataSync_ = dataSync; - return E_OK; -} - -int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync) -{ - int errCode = ParamCheck(mode, context, dataSync); - if (errCode != E_OK) { - return errCode; - } - errCode = dataSync->CheckPermitSendData(mode, context); - if (errCode != E_OK) { - return errCode; - } - std::lock_guard lock(lock_); - if (sessionId_ != 0) { // auto sync timeout resend - return ReSend(); - } - Init(mode, context, dataSync); - LOGI("[slws] SendStart,mode=%d,label=%s,device=%s", mode_, dataSync->GetLabel().c_str(), - STR_MASK(context->GetDeviceId())); - int tmpMode = SyncOperation::TransferSyncMode(mode); - if (tmpMode == SyncModeType::PUSH) { - errCode = dataSync->PushStart(context); - } else if (tmpMode == SyncModeType::PUSH_AND_PULL) { - errCode = dataSync->PushPullStart(context); - } else if (tmpMode == SyncModeType::PULL) { - errCode = dataSync->PullRequestStart(context); - } else { - errCode = dataSync->PullResponseStart(context); - } - if (context->IsSkipTimeoutError(errCode)) { - // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. - // just return to avoid higher pressure for send. - UpdateInfo(); - return errCode; - } - if (errCode != E_OK) { - LOGE("[slws] SendStart errCode=%d", errCode); - return errCode; - } - errCode = UpdateInfo(); - if (tmpMode == SyncModeType::PUSH_AND_PULL && context_->GetTaskErrCode() == -E_EKEYREVOKED) { - LOGE("wait for recv finished for push and pull mode"); - return -E_EKEYREVOKED; - } - if (errCode == -E_FINISHED) { - return E_OK; - } - return InnerSend(); -} - -int SlidingWindowSender::UpdateInfo() -{ - ReSendInfo reSendInfo; - SyncTimeRange reSendDataTimeRange = context_->GetDataTimeRange(); - reSendInfo.start = reSendDataTimeRange.beginTime; - reSendInfo.end = reSendDataTimeRange.endTime; - reSendInfo.deleteBeginTime = reSendDataTimeRange.deleteBeginTime; - reSendInfo.deleteEndTime = reSendDataTimeRange.deleteEndTime; - reSendInfo.packetId = context_->GetPacketId(); - maxSequenceIdhasSent_++; - reSendMap_[maxSequenceIdhasSent_] = reSendInfo; - windowSize_--; - LOGI("[slws] mode=%d,start=%llu,end=%llu,deleteStart=%llu,deleteEnd=%llu,seqId=%d,packetId=%llu,window_size=%d," - "label=%s,device=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, - reSendInfo.deleteEndTime, maxSequenceIdhasSent_, reSendInfo.packetId, windowSize_, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - ContinueToken token; - context_->GetContinueToken(token); - if (token == nullptr) { - isAllDataHasSent_ = true; - LOGI("[slws] UpdateInfo all data has sent"); - return -E_FINISHED; - } - return E_OK; -} - -int SlidingWindowSender::InnerSend() -{ - while (true) { - if (windowSize_ <= 0 || isAllDataHasSent_) { - LOGI("[slws] InnerSend windowSize_=%d,isAllDataHasSent_=%d", windowSize_, isAllDataHasSent_); - return E_OK; - } - if (isErr_ || context_ == nullptr) { - LOGI("[slws] InnerSend isErr"); - return -E_SLIDING_WINDOW_SENDER_ERR; - } - int errCode; - context_->IncSequenceId(); - int mode = SyncOperation::TransferSyncMode(mode_); - if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { - errCode = dataSync_->PushStart(context_); - } else { - errCode = dataSync_->PullResponseStart(context_); - } - if ((mode == SyncModeType::PUSH_AND_PULL) && errCode == -E_EKEYREVOKED) { - LOGE("errCode = %d, wait for recv finished for push and pull mode", errCode); - isAllDataHasSent_ = true; - return -E_EKEYREVOKED; - } - if (context_->IsSkipTimeoutError(errCode)) { - // if E_TIMEOUT occurred, means send message pressure is high, put into resend map and wait for resend. - // just return to avoid higher pressure for send. - UpdateInfo(); - return errCode; - } - if (errCode != E_OK) { - isErr_ = true; - LOGE("[slws] InnerSend errCode=%d", errCode); - return errCode; - } - errCode = UpdateInfo(); - if (errCode == -E_FINISHED) { - return E_OK; - } - } -} - -int SlidingWindowSender::PreHandleAckRecv(const Message *message) -{ - if (message == nullptr) { - LOGE("[slws] AckRecv message nullptr"); - return -E_INVALID_ARGS; - } - if (message->GetMessageType() == TYPE_NOTIFY || message->IsFeedbackError()) { - return E_OK; - } - const DataAckPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value - std::lock_guard lock(lock_); - uint32_t sequenceId = message->GetSequenceId(); - if (reSendMap_.count(sequenceId) != 0) { - uint64_t originalPacketId = reSendMap_[sequenceId].packetId; - if (DataAckPacket::IsPacketIdValid(packetId) && packetId != originalPacketId) { - LOGE("[slws] packetId[%llu] is not match with original[%llu]", packetId, originalPacketId); - return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; - } - } - return E_OK; -} - -int SlidingWindowSender::AckRecv(const Message *message) -{ - if (message == nullptr) { - LOGE("[slws] AckRecv message nullptr"); - return -E_INVALID_ARGS; - } - const DataAckPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value - uint32_t sessionId = message->GetSessionId(); - uint32_t sequenceId = message->GetSequenceId(); - LOGI("[slws] AckRecv sequecneId=%d,packetId=%llu,label=%s,dev=%s", sequenceId, packetId, - dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); - - std::lock_guard lock(lock_); - if (sessionId != sessionId_) { - LOGI("[slws] AckRecv sessionId is different"); - return E_OK; - } - if (reSendMap_.count(sequenceId) != 0) { - reSendMap_.erase(sequenceId); - windowSize_++; - LOGI("[slws] AckRecv window_size=%d", windowSize_); - } else { - LOGI("[slws] AckRecv sequenceId not in map"); - return E_OK; - } - if (!isAllDataHasSent_) { - return InnerSend(); - } else if (reSendMap_.size() == 0) { - context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); - LOGI("[slws] AckRecv all finished,label=%s,dev=%s", dataSync_->GetLabel().c_str(), - STR_MASK(context_->GetDeviceId())); - InnerClear(); - return -E_FINISHED; - } - return E_OK; -} - -int SlidingWindowSender::ReSend() const -{ - if (isErr_) { - LOGI("[slws] ReSend InnerSend isErr"); - return -E_INTERNAL_ERROR; - } - if (reSendMap_.empty()) { - LOGI("[slws] ReSend map empty"); - return -E_INTERNAL_ERROR; - } - uint32_t sequenceId = reSendMap_.begin()->first; - ReSendInfo reSendInfo = reSendMap_.begin()->second; - LOGI("[slws] ReSend mode=%d,start=%llu,end=%llu,delStart=%llu,delEnd=%llu,seqId=%d,packetId=%llu,windowsize=%d," - "label=%s,deviceId=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, - reSendInfo.deleteEndTime, sequenceId, reSendInfo.packetId, windowSize_, dataSync_->GetLabel().c_str(), - STR_MASK(context_->GetDeviceId())); - DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, - reSendInfo.deleteBeginTime, reSendInfo.deleteEndTime, reSendInfo.packetId}; - return dataSync_->ReSend(context_, dataReSendInfo); -} - -void SlidingWindowSender::Clear() -{ - std::lock_guard lock(lock_); - LOGD("[slws] clear sender info"); - InnerClear(); -} - -void SlidingWindowSender::InnerClear() -{ - sessionId_ = 0; - reSendMap_.clear(); - windowSize_ = 0; - maxSequenceIdhasSent_ = 0; - isAllDataHasSent_ = false; - isErr_ = false; -} - -void SlidingWindowSender::SetErr(bool isErr) -{ - std::lock_guard lock(lock_); - isErr_ = isErr; -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h deleted file mode 100755 index 652a20a1812bdd8e3d60e82c2b512d0c6ffad387..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SLIDING_WINDOW_SENDER_H -#define SLIDING_WINDOW_SENDER_H -#include - -#include "message.h" -#include "single_ver_data_sync.h" -#include "single_ver_sync_task_context.h" - -namespace DistributedDB { -struct ReSendInfo { - TimeStamp start = 0; - TimeStamp end = 0; - TimeStamp deleteBeginTime = 0; - TimeStamp deleteEndTime = 0; - // packetId is used for matched ackpacket packetId which saved in ackPacket.reserve - // if equaled, means need to handle the ack, or drop. it is always increased - uint64_t packetId = 0; -}; - -class SlidingWindowSender { -public: - SlidingWindowSender() = default; - ~SlidingWindowSender(); - DISABLE_COPY_ASSIGN_MOVE(SlidingWindowSender); - - int Initialize(SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - // start one dataSync, mode can be push pushAndPull or pullResponse - int SendStart(int32_t mode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int AckRecv(const Message *message); - int PreHandleAckRecv(const Message *message); - void Clear(); - void SetErr(bool isErr); -private: - int ParamCheck(int32_t mode, const SingleVerSyncTaskContext *context, - std::shared_ptr &dataSync); - void Init(int32_t inMode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - int UpdateInfo(); - int InnerSend(); - void InnerClear(); - int ReSend() const; // when timeout, used for reSend data - - // initial max window size - static const int MAX_WINDOW_SIZE = 3; - std::mutex lock_; - // 0 is default invalid sessionId. - uint32_t sessionId_ = 0; - // the sequenceId as key - std::map reSendMap_; - // remaining sending window - int32_t windowSize_ = 0; - // in this session, the max sequenceId has been sent - uint32_t maxSequenceIdhasSent_ = 0; - // int this session, all data has been sent, but all ack has received or not - bool isAllDataHasSent_ = false; - SingleVerSyncTaskContext *context_ = nullptr; - std::shared_ptr dataSync_ = nullptr; - int mode_ = 0; // sync mode, e.g. push - bool isErr_ = true; -}; -} // namespace DistributedDB -#endif // SLIDING_WINDOW_SENDER_H - diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp index 413d37543a0d6bbc094d8c1d9ea774abc34e8e48..d07877666cd05c140402c4fb8e67a715b555378f 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp @@ -93,10 +93,7 @@ int SyncStateMachine::TimeoutCallback(TimerId timerId) } retryTime++; syncContext_->SetRetryTime(retryTime); - // when version is high than 102, the sequenceid will be managed by slide windows sender. - if (syncContext_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - syncContext_->IncSequenceId(); - } + // the sequenceid will be managed by dataSync slide windows. syncContext_->SetRetryStatus(SyncTaskContext::NEED_RETRY); int timeoutTime = syncContext_->GetSyncRetryTimeout(retryTime); syncContext_->ModifyTimer(timeoutTime); @@ -125,12 +122,6 @@ void SyncStateMachine::Abort() int SyncStateMachine::SwitchMachineState(uint8_t event) { - if (syncContext_->GetRemoteSoftwareVersion() == SOFTWARE_VERSION_EARLIEST) { - currentSyncProctolVersion_ = SINGLE_VER_SYNC_PROCTOL_V2; // ver 101 102 use same table - } else if (syncContext_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_EARLIEST) { - currentSyncProctolVersion_ = syncContext_->GetRemoteSoftwareVersion(); - } - const std::vector &tables = GetStateSwitchTables(); auto tableIter = std::find_if(tables.begin(), tables.end(), [this](const StateSwitchTable &table) { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp index 099a76277aa74bbe22a2f4ccc010534f62931592..909d78c58abb1b42b27905e8706f7e8cf1611660 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp @@ -109,8 +109,8 @@ void SyncTaskContext::SetOperationStatus(int status) LOGD("[SyncTaskContext][SetStatus] syncOperation is null"); return; } - int finalStatus = status; + int operationStatus = syncOperation_->GetStatus(deviceId_); if (status == SyncOperation::OP_SEND_FINISHED && operationStatus == SyncOperation::OP_RECV_FINISHED) { if (GetTaskErrCode() == -E_EKEYREVOKED) { diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index 7b98fee3f510bb675998ff3cafe223706c0199cf..0a46a901d0de9dd55fbc67ebd04a8f49aefd6ec7 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -25,14 +25,18 @@ config("module_private_config") { "./unittest/common/interfaces", "../include", "../interfaces/include", + "../interfaces/include/relational", "../interfaces/src", + "../interfaces/src/relational", "../storage/include", "../storage/src", "../storage/src/multiver", "../storage/src/operation", "../storage/src/sqlite", + "../storage/src/sqlite/relational", "../storage/src/upgrader", "../common/include", + "../common/include/relational", "../common/src", "../communicator/include", "../communicator/src", @@ -53,6 +57,7 @@ config("module_private_config") { "LOW_LEVEL_MEM_DEV", "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", + "RELATIONAL_STORE", ] } @@ -85,6 +90,7 @@ ohos_source_set("src_file") { "../common/src/query.cpp", "../common/src/query_expression.cpp", "../common/src/ref_object.cpp", + "../common/src/relational/relational_schema_object.cpp", "../common/src/runtime_context.cpp", "../common/src/runtime_context_impl.cpp", "../common/src/schema_object.cpp", @@ -118,6 +124,13 @@ ohos_source_set("src_file") { "../interfaces/src/kv_store_nb_delegate_impl.cpp", "../interfaces/src/kv_store_result_set_impl.cpp", "../interfaces/src/kv_store_snapshot_delegate_impl.cpp", + "../interfaces/src/relational/data_value.cpp", + "../interfaces/src/relational/relational_store_delegate_impl.cpp", + "../interfaces/src/relational/relational_store_manager.cpp", + "../interfaces/src/relational/relational_store_sqlite_ext.cpp", + "../interfaces/src/relational/runtime_config.cpp", + "../storage/src/data_transformer.cpp", + "../storage/src/db_properties.cpp", "../storage/src/default_factory.cpp", "../storage/src/generic_kvdb.cpp", "../storage/src/generic_kvdb_connection.cpp", @@ -148,10 +161,17 @@ ohos_source_set("src_file") { "../storage/src/operation/multi_ver_database_oper.cpp", "../storage/src/operation/single_ver_database_oper.cpp", "../storage/src/package_file.cpp", + "../storage/src/relational_store_connection.cpp", + "../storage/src/relational_store_instance.cpp", + "../storage/src/relational_sync_able_storage.cpp", + "../storage/src/relationaldb_properties.cpp", "../storage/src/result_entries_window.cpp", "../storage/src/single_ver_natural_store_commit_notify_data.cpp", "../storage/src/sqlite/query_object.cpp", "../storage/src/sqlite/query_sync_object.cpp", + "../storage/src/sqlite/relational/sqlite_relational_store.cpp", + "../storage/src/sqlite/relational/sqlite_relational_store_connection.cpp", + "../storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp", "../storage/src/sqlite/sqlite_local_kvdb.cpp", "../storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "../storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -165,6 +185,8 @@ ohos_source_set("src_file") { "../storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp", + "../storage/src/sqlite/sqlite_single_ver_relational_continue_token.cpp", + "../storage/src/sqlite/sqlite_single_ver_relational_storage_executor.cpp", "../storage/src/sqlite/sqlite_single_ver_result_set.cpp", "../storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.cpp", "../storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", @@ -177,6 +199,7 @@ ohos_source_set("src_file") { "../storage/src/storage_engine.cpp", "../storage/src/storage_engine_manager.cpp", "../storage/src/storage_executor.cpp", + "../storage/src/sync_able_engine.cpp", "../storage/src/sync_able_kvdb.cpp", "../storage/src/sync_able_kvdb_connection.cpp", "../storage/src/upgrader/single_ver_database_upgrader.cpp", @@ -193,18 +216,17 @@ ohos_source_set("src_file") { "../syncer/src/multi_ver_sync_task_context.cpp", "../syncer/src/multi_ver_syncer.cpp", "../syncer/src/query_sync_water_mark_helper.cpp", + "../syncer/src/single_ver_data_message_schedule.cpp", "../syncer/src/single_ver_data_packet.cpp", "../syncer/src/single_ver_data_sync.cpp", - "../syncer/src/single_ver_data_sync_with_sliding_window.cpp", "../syncer/src/single_ver_kv_syncer.cpp", + "../syncer/src/single_ver_relational_syncer.cpp", "../syncer/src/single_ver_serialize_manager.cpp", "../syncer/src/single_ver_sync_engine.cpp", "../syncer/src/single_ver_sync_state_machine.cpp", "../syncer/src/single_ver_sync_target.cpp", "../syncer/src/single_ver_sync_task_context.cpp", "../syncer/src/single_ver_syncer.cpp", - "../syncer/src/sliding_window_receiver.cpp", - "../syncer/src/sliding_window_sender.cpp", "../syncer/src/subscribe_manager.cpp", "../syncer/src/sync_engine.cpp", "../syncer/src/sync_operation.cpp", @@ -362,8 +384,10 @@ distributeddb_unittest("DistributedDBSingleVerP2PSyncTest") { [ "unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp" ] } -distributeddb_unittest("DistributedDBSyncModuleTest") { - sources = [ "unittest/common/syncer/distributeddb_sync_module_test.cpp" ] +distributeddb_unittest("DistributedDBSingleVerMsgScheduleTest") { + sources = [ + "unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp", + ] } distributeddb_unittest("DistributedDBInterfacesNBDelegateTest") { @@ -643,6 +667,7 @@ group("unittest") { ":DistributedDBParcelTest", ":DistributedDBSchemaObjectTest", ":DistributedDBSchemalTest", + ":DistributedDBSingleVerMsgScheduleTest", ":DistributedDBSingleVerP2PQuerySyncTest", ":DistributedDBSingleVerP2PSubscribeSyncTest", ":DistributedDBSingleVerP2PSyncCheckTest", @@ -661,7 +686,6 @@ group("unittest") { ":DistributedDBStorageSingleVerUpgradeTest", ":DistributedDBStorageTransactionDataTest", ":DistributedDBStorageTransactionRecordTest", - ":DistributedDBSyncModuleTest", ":DistributedDBSyncerDeviceManagerTest", ":DistributedDBTimeSyncTest", ":RuntimeContextProcessSystemApiAdapterImplTest", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..47c10ebe9587fd705448616aa3a235cd91b037a1 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_relational_get_data_test.cpp @@ -0,0 +1,395 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include + +#include "data_transformer.h" +#include "db_common.h" +#include "db_constant.h" +#include "db_errno.h" +#include "db_types.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "generic_single_ver_kv_entry.h" +#include "kvdb_properties.h" +#include "log_print.h" +#include "relational_store_delegate.h" +#include "relational_store_instance.h" +#include "relational_store_manager.h" +#include "relational_store_sqlite_ext.h" +#include "relational_sync_able_storage.h" +#include "sqlite_relational_store.h" +#include "sqlite_utils.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { +string g_testDir; +string g_storePath; +string g_storeID = "dftStoreID"; +const string g_tableName { "data" }; +DistributedDB::RelationalStoreManager g_mgr(APP_ID, USER_ID); +RelationalStoreDelegate *g_delegate = nullptr; + +void CreateDBAndTable() +{ + sqlite3 *db = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + sqlite3_close(db); + return; + } + + const string sql = + "PRAGMA journal_mode=WAL;" + "CREATE TABLE " + g_tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; + char *zErrMsg = nullptr; + errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &zErrMsg); + if (errCode != SQLITE_OK) { + LOGE("sql error:%s", zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_close(db); +} + +int AddOrUpdateRecord(int64_t key, int64_t value) +{ + sqlite3 *db = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode == SQLITE_OK) { + const string sql = + "INSERT OR REPLACE INTO " + g_tableName + " VALUES(" + to_string(key) + "," + to_string(value) + ");"; + errCode = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr); + } + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + sqlite3_close(db); + return errCode; +} + +int GetLogData(int key, uint64_t &flag, TimeStamp ×tamp, const DeviceID &device = "") +{ + string tableName = g_tableName; + if (!device.empty()) { + } + const string sql = "SELECT timestamp, flag \ + FROM " + g_tableName + " as a, " + DBConstant::RELATIONAL_PREFIX + g_tableName + "_log as b \ + WHERE a.key=? AND a.rowid=b.data_key;"; + + sqlite3 *db = nullptr; + sqlite3_stmt *statement = nullptr; + int errCode = sqlite3_open(g_storePath.c_str(), &db); + if (errCode != SQLITE_OK) { + LOGE("open db failed:%d", errCode); + errCode = SQLiteUtils::MapSQLiteErrno(errCode); + goto END; + } + errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::BindInt64ToStatement(statement, 1, key); // 1 means key's index + if (errCode != E_OK) { + goto END; + } + errCode = SQLiteUtils::StepWithRetry(statement, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + timestamp = static_cast(sqlite3_column_int64(statement, 0)); + flag = static_cast(sqlite3_column_int64(statement, 1)); + errCode = E_OK; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = -E_NOT_FOUND; + } + +END: + sqlite3_close(db); + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +void InitStoreProp(const std::string &storePath, const std::string &appId, const std::string &userId, + RelationalDBProperties &properties) +{ + properties.SetStringProp(RelationalDBProperties::DATA_DIR, storePath); + properties.SetStringProp(RelationalDBProperties::APP_ID, appId); + properties.SetStringProp(RelationalDBProperties::USER_ID, userId); + properties.SetStringProp(RelationalDBProperties::STORE_ID, g_storeID); + std::string identifier = userId + "-" + appId + "-" + g_storeID; + std::string hashIdentifier = DBCommon::TransferHashString(identifier); + properties.SetStringProp(RelationalDBProperties::IDENTIFIER_DATA, hashIdentifier); +} + +const RelationalSyncAbleStorage *GetRelationalStore() +{ + RelationalDBProperties properties; + InitStoreProp(g_storePath, APP_ID, USER_ID, properties); + int errCode = E_OK; + auto store = RelationalStoreInstance::GetDataBase(properties, errCode); + if (store == nullptr) { + LOGE("Get db failed:%d", errCode); + return nullptr; + } + return static_cast(store)->GetStorageEngine(); +} + +int GetCount(sqlite3 *db, const string &sql, size_t &count) +{ + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + return errCode; + } + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + count = static_cast(sqlite3_column_int64(stmt, 0)); + errCode = E_OK; + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} +} + +class DistributedDBRelationalGetDataTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBRelationalGetDataTest::SetUpTestCase(void) +{ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + g_storePath = g_testDir + "/getDataTest.db"; + LOGI("The test db is:%s", g_testDir.c_str()); +} + +void DistributedDBRelationalGetDataTest::TearDownTestCase(void) +{} + +void DistributedDBRelationalGetDataTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + CreateDBAndTable(); +} + +void DistributedDBRelationalGetDataTest::TearDown(void) +{ + if (g_delegate != nullptr) { + EXPECT_EQ(g_mgr.CloseStore(g_delegate), DBStatus::OK); + EXPECT_EQ(g_mgr.DeleteStore(g_storePath), DBStatus::OK); + g_delegate = nullptr; + } + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error."); + } + return; +} + +/** + * @tc.name: LogTbl1 + * @tc.desc: When put sync data to relational store, trigger generate log. + * @tc.type: FUNC + * @tc.require: AR000GK58G + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, LogTbl1, TestSize.Level1) +{ + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); + + /** + * @tc.steps: step1. Put data. + * @tc.expected: Succeed, return OK. + */ + int insertKey = 1; + int insertValue = 1; + EXPECT_EQ(AddOrUpdateRecord(insertKey, insertValue), E_OK); + + /** + * @tc.steps: step2. Check log record. + * @tc.expected: Record exists. + */ + uint64_t flag = 0; + TimeStamp timestamp1 = 0; + EXPECT_EQ(GetLogData(insertKey, flag, timestamp1), E_OK); + EXPECT_EQ(flag, DataItem::LOCAL_FLAG); + EXPECT_NE(timestamp1, 0ULL); +} + +/** + * @tc.name: GetSyncData1 + * @tc.desc: GetSyncData interface + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetSyncData1, TestSize.Level1) +{ + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); + + /** + * @tc.steps: step1. Put 500 records. + * @tc.expected: Succeed, return OK. + */ + const int RECORD_COUNT = 500; + for (int i = 0; i < RECORD_COUNT; ++i) { + EXPECT_EQ(AddOrUpdateRecord(i, i), E_OK); + } + + /** + * @tc.steps: step2. Get all data. + * @tc.expected: Succeed and the count is right. + */ + auto store = GetRelationalStore(); + ContinueToken token = nullptr; + QueryObject query(Query::Select(g_tableName)); + std::vector entries; + DataSizeSpecInfo sizeInfo {MTU_SIZE, 50}; + + int errCode = store->GetSyncData(query, SyncTimeRange {}, sizeInfo, token, entries); + int count = entries.size(); + SingleVerKvEntry::Release(entries); + EXPECT_EQ(errCode, -E_UNFINISHED); + while (token != nullptr) { + errCode = store->GetSyncDataNext(entries, token, sizeInfo); + count += entries.size(); + SingleVerKvEntry::Release(entries); + EXPECT_TRUE(errCode == E_OK || errCode == -E_UNFINISHED); + } + EXPECT_EQ(count, RECORD_COUNT); +} + +/** + * @tc.name: GetQuerySyncData1 + * @tc.desc: GetSyncData interface. + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetQuerySyncData1, TestSize.Level1) +{ + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); + + /** + * @tc.steps: step1. Put 100 records. + * @tc.expected: Succeed, return OK. + */ + const int RECORD_COUNT = 100; // 100 records. + for (int i = 0; i < RECORD_COUNT; ++i) { + EXPECT_EQ(AddOrUpdateRecord(i, i), E_OK); + } + + /** + * @tc.steps: step2. Get data limit 80, offset 30. + * @tc.expected: Get 70 records. + */ + auto store = GetRelationalStore(); + ContinueToken token = nullptr; + const unsigned int LIMIT = 80; // limit as 80. + const unsigned int OFFSET = 30; // offset as 30. + const unsigned int EXPECT_COUNT = RECORD_COUNT - OFFSET; // expect 70 records. + QueryObject query(Query::Select(g_tableName).Limit(LIMIT, OFFSET)); + std::vector entries; + + int errCode = store->GetSyncData(query, SyncTimeRange {}, DataSizeSpecInfo {}, token, entries); + EXPECT_EQ(entries.size(), EXPECT_COUNT); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(token, nullptr); + SingleVerKvEntry::Release(entries); +} + +/** + * @tc.name: GetIncorrectTypeData1 + * @tc.desc: GetSyncData and PutSyncDataWithQuery interface. + * @tc.type: FUNC + * @tc.require: AR000GK58H + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBRelationalGetDataTest, GetIncorrectTypeData1, TestSize.Level1) +{ + ASSERT_EQ(g_mgr.OpenStore(g_storePath, g_storeID, RelationalStoreDelegate::Option {}, g_delegate), DBStatus::OK); + ASSERT_NE(g_delegate, nullptr); + ASSERT_EQ(g_delegate->CreateDistributedTable(g_tableName), DBStatus::OK); + + /** + * @tc.steps: step1. Create distributed table "dataPlus". + * @tc.expected: Succeed, return OK. + */ + sqlite3 *db = nullptr; + EXPECT_EQ(sqlite3_open(g_storePath.c_str(), &db), SQLITE_OK); + const string tableName = g_tableName + "Plus"; + string sql = "CREATE TABLE " + tableName + "(key INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, value INTEGER);"; + ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + ASSERT_EQ(g_delegate->CreateDistributedTable(tableName), DBStatus::OK); + + /** + * @tc.steps: step2. Put 5 records with different type into "dataPlus" table. + * @tc.expected: Succeed, return OK. + */ + vector sqls = { + "INSERT INTO " + tableName + " VALUES(NULL, 1);", + "INSERT INTO " + tableName + " VALUES(NULL, 0.01);", + "INSERT INTO " + tableName + " VALUES(NULL, NULL);", + "INSERT INTO " + tableName + " VALUES(NULL, 'This is a text.');", + "INSERT INTO " + tableName + " VALUES(NULL, x'0123456789');", + }; + const size_t RECORD_COUNT = sqls.size(); + for (const auto &sql : sqls) { + ASSERT_EQ(sqlite3_exec(db, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + } + + /** + * @tc.steps: step3. Get all data from "dataPlus" table. + * @tc.expected: Succeed and the count is right. + */ + auto store = GetRelationalStore(); + ContinueToken token = nullptr; + QueryObject query(Query::Select(tableName)); + std::vector entries; + EXPECT_EQ(store->GetSyncData(query, SyncTimeRange {}, DataSizeSpecInfo {}, token, entries), E_OK); + EXPECT_EQ(entries.size(), RECORD_COUNT); + + /** + * @tc.steps: step4. Put data into "data" table from deviceA. + * @tc.expected: Succeed, return OK. + */ + QueryObject queryPlus(Query::Select(g_tableName)); + const DeviceID deviceID = "deviceA"; + EXPECT_EQ(const_cast(store)->PutSyncDataWithQuery(queryPlus, entries, deviceID), E_OK); + SingleVerKvEntry::Release(entries); + + /** + * @tc.steps: step5. Check data. + * @tc.expected: All data in the two tables are same. + */ + sql = "SELECT count(*) " + "FROM " + tableName + " as a, " + DBConstant::RELATIONAL_PREFIX + g_tableName + "_" + + DBCommon::TransferStringToHex(DBCommon::TransferHashString(deviceID)) + " as b " + "WHERE a.key=b.key AND (a.value=b.value OR (a.value is NULL AND b.value is NULL));"; + size_t count = 0; + EXPECT_EQ(GetCount(db, sql, count), E_OK); + EXPECT_EQ(count, RECORD_COUNT); + sqlite3_close(db); +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp index 50496bb84b42b6ca7ce06f88473972954df70f75..f8c8df347e07489c5afd03f66cfb1116ea947263 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp @@ -64,7 +64,7 @@ namespace { std::string GetKvStoreDirectory(const std::string &userId, const std::string &appId, const std::string &storeId) { - string identifier = userId + "-" + appId + "-" + storeId; + string identifier = DBCommon::GenerateIdentifierId(storeId, appId, userId); string hashIdentifierName = DBCommon::TransferHashString(identifier); string identifierName = DBCommon::TransferStringToHex(hashIdentifierName); string filePath = g_testDir + "/" + identifierName + "/" + DBConstant::SINGLE_SUB_DIR + "/main/"; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp index c24f02a1e3bd7619a22ff6498271455bb5342c71..a9f55059bf51f10f09ef160631231df1cd8939f6 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -61,6 +61,7 @@ namespace { } EXPECT_EQ(SQLiteUtils::RegisterCalcHash(db), E_OK); EXPECT_EQ(SQLiteUtils::RegisterGetSysTime(db), E_OK); + EXPECT_EQ(sqlite3_exec(db, "PRAGMA journal_mode=WAL;", nullptr, nullptr, nullptr), SQLITE_OK); return rc; } @@ -148,21 +149,12 @@ namespace { { std::vector devices = {DEVICE_B}; Query query = Query::Select(g_tableName); - std::mutex syncLock; - std::condition_variable syncCondVar; std::map> statusMap; - std::mutex syncMutex; - std::condition_variable syncCv; - SyncStatusCallback callBack = [&statusMap, &syncCv]( + SyncStatusCallback callBack = [&statusMap]( const std::map> &devicesMap) { statusMap = devicesMap; - syncCv.notify_one(); }; - DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, callBack, query, false); - std::unique_lock uniqueLock(syncMutex); - syncCv.wait(uniqueLock, [&statusMap]() { - return !statusMap.empty(); - }); + DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, query, callBack, true); EXPECT_EQ(callStatus, OK); for (const auto &tablesRes : statusMap) { for (const auto &tableStatus : tablesRes.second) { @@ -185,7 +177,7 @@ namespace { EXPECT_EQ(GetDB(db), SQLITE_OK); EXPECT_EQ(CreateTable(db), SQLITE_OK); - EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName, {}), OK); + EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName), OK); sqlite3_close(db); @@ -230,6 +222,9 @@ void DistributedDBRelationalVerP2PSyncTest::SetUpTestCase() */ DistributedDBToolsUnitTest::TestDirInit(g_testDir); g_dbDir = g_testDir + "/test.db"; + sqlite3 *db = nullptr; + ASSERT_EQ(GetDB(db), SQLITE_OK); + sqlite3_close(db); g_communicatorAggregator = new (std::nothrow) VirtualCommunicatorAggregator(); ASSERT_TRUE(g_communicatorAggregator != nullptr); @@ -254,7 +249,7 @@ void DistributedDBRelationalVerP2PSyncTest::SetUp(void) /** * @tc.setup: create virtual device B, and get a KvStoreNbDelegate as deviceA */ - g_mgr.OpenStore(g_dbDir, {true}, g_kvDelegateCallback); + g_kvDelegateStatus = g_mgr.OpenStore(g_dbDir, "Relational_default_id", {}, g_kvDelegatePtr); ASSERT_TRUE(g_kvDelegateStatus == OK); ASSERT_TRUE(g_kvDelegatePtr != nullptr); g_deviceB = new (std::nothrow) RelationalVirtualDevice(DEVICE_B); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c5c0afa8d159d2cc4b5c268b4cfa449bb3ce3995 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_msg_schedule_test.cpp @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "distributeddb_tools_unit_test.h" +#include "single_ver_data_message_schedule.h" +#include "single_ver_data_packet.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { +} + +class DistributedDBSingleVerMsgScheduleTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBSingleVerMsgScheduleTest::SetUpTestCase(void) +{ +} + +void DistributedDBSingleVerMsgScheduleTest::TearDownTestCase(void) +{ +} + +void DistributedDBSingleVerMsgScheduleTest::SetUp(void) +{ +} + +void DistributedDBSingleVerMsgScheduleTest::TearDown(void) +{ +} + +/** + * @tc.name: MsgSchedule001 + * @tc.desc: Test MsgSchedule funtion with normal sequenceId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule001, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg sequence_3, sequence_2, sequence_1 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 3; i >= 1; i--) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + if (i > 1) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + } + } + /** + * @tc.steps: step2. get msg + * @tc.expected: get msg by sequence_1, sequence_2, sequence_3 + */ + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule002 + * @tc.desc: Test MsgSchedule funtion with by low version + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule002, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg session1_sequence1, session2_sequence1 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_RELEASE_2_0); + DataSyncMessageInfo info; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sessionId_ = i; + info.sequenceId_ = 1; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + /** + * @tc.steps: step2. get msg + * @tc.expected: get msg by session2_sequence1 + */ + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), 1u); + msgSchedule.ScheduleInfoHandle(false, false, msg); + delete msg; + msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule003 + * @tc.desc: Test MsgSchedule funtion with cross sessionId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule003, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg session1_seq1, session2_seq1, session1_seq2, session2_seq2, and handle session1_seq1 + * @tc.expected: handle ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + bool isNeedHandle = true; + bool isNeedContinue = true; + info.sessionId_ = 1; + info.sequenceId_ = 1; + info.packetId_ = 1; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + info.sessionId_ = 2; + info.sequenceId_ = 1; + info.packetId_ = 1; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + info.sessionId_ = 1; + info.sequenceId_ = 2; + info.packetId_ = 2; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + info.sessionId_ = 2; + info.sequenceId_ = 2; + info.packetId_ = 2; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + + /** + * @tc.steps: step2. get msg + * @tc.expected: get msg by session2_seq1, session2_seq2 + */ + for (uint32_t i = 1; i <= 2; i++) { + msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + EXPECT_EQ(msg->GetSessionId(), 2u); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule004 + * @tc.desc: Test MsgSchedule funtion with same sessionId with different packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule004, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq2_packet2, seq3_packet3, seq1_packet4, seq2_packet5, seq3_packet6 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 2; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + } + for (uint32_t i = 1; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i + 3; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + /** + * @tc.steps: step2. get msg + * @tc.expected: drop seq2_packet2, seq3_packet3 and get seq1_packet4, seq2_packet5, seq3_packet6 + */ + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i + 3); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule005 + * @tc.desc: Test MsgSchedule funtion with same sessionId with different packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule005, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq1_packet4, seq2_packet5, seq2_packet2, seq3_packet3 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i + 3; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + for (uint32_t i = 2; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + /** + * @tc.steps: step2. get msg + * @tc.expected: drop seq2_packet2, seq3_packet3 and get seq1_packet4, seq2_packet5 + */ + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i + 3); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule006 + * @tc.desc: Test MsgSchedule funtion with same sessionId and same sequenceId and packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule006, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq1_packet1, seq2_packet2 + * @tc.expected: put msg ok + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + for (uint32_t i = 1; i <= 2; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + /** + * @tc.steps: step2. put msg seq1_packet1, seq2_packet2 again and seq3_packet3 + * @tc.expected: get msg ok and get seq1_packet1, seq2_packet2 and seq3_packet3 + */ + for (uint32_t i = 1; i <= 3; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, (i == 3) ? true : false); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} + +/** + * @tc.name: MsgSchedule007 + * @tc.desc: Test MsgSchedule funtion with same sessionId and duplicate sequenceId and low packetId + * @tc.type: FUNC + * @tc.require: + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerMsgScheduleTest, MsgSchedule007, TestSize.Level0) +{ + /** + * @tc.steps: step1. put msg seq1_packet4, seq2_packet5 + * @tc.expected: put msg ok and get msg seq1_packet4, seq2_packet5 + */ + SingleVerDataMessageSchedule msgSchedule; + auto *context = new SingleVerSyncTaskContext(); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_CURRENT); + DataSyncMessageInfo info; + info.sessionId_ = 10; + bool isNeedHandle = true; + bool isNeedContinue = true; + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i + 3; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + for (uint32_t i = 1; i <= 2; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg != nullptr); + EXPECT_EQ(isNeedContinue, true); + EXPECT_EQ(isNeedHandle, true); + EXPECT_EQ(msg->GetSequenceId(), i); + const DataRequestPacket *packet = msg->GetObject(); + EXPECT_EQ(packet->GetPacketId(), i + 3); + msgSchedule.ScheduleInfoHandle(isNeedHandle, false, msg); + delete msg; + } + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + ASSERT_TRUE(msg == nullptr); + /** + * @tc.steps: step2. put msg seq1_packet1, seq2_packet2 + * @tc.expected: get nullptr + */ + for (uint32_t i = 1; i <= 2; i++) { + info.sequenceId_ = i; + info.packetId_ = i; + DistributedDB::Message *message = nullptr; + DistributedDBToolsUnitTest::BuildMessage(info, message); + msgSchedule.PutMsg(message); + } + isNeedHandle = true; + isNeedContinue = true; + for (uint32_t i = 1; i <= 3; i++) { + Message *msg = msgSchedule.MoveNextMsg(context, isNeedHandle, isNeedContinue); + EXPECT_EQ(isNeedContinue, true); + ASSERT_TRUE(msg == nullptr); + } + RefObject::KillAndDecObjRef(context); + context = nullptr; +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp deleted file mode 100644 index 2fc18aa7e61d52d5a6ac3eeb731a7678ad4bbf8d..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "db_constant.h" -#include "distributeddb_tools_unit_test.h" -#include "mock_communicator.h" -#include "mock_single_ver_data_sync.h" -#include "mock_single_ver_sync_task_context.h" -#include "sliding_window_receiver.h" - -using namespace testing::ext; -using namespace DistributedDB; -using namespace DistributedDBUnitTest; - -class DistributedDBSyncModuleTest : public testing::Test { -public: - static void SetUpTestCase(void); - static void TearDownTestCase(void); - void SetUp(); - void TearDown(); -}; - -void DistributedDBSyncModuleTest::SetUpTestCase(void) -{ -} - -void DistributedDBSyncModuleTest::TearDownTestCase(void) -{ -} - -void DistributedDBSyncModuleTest::SetUp(void) -{ -} - -void DistributedDBSyncModuleTest::TearDown(void) -{ -} - - -namespace { - void InitMessage(DataSyncMessageInfo &info) - { - info.messageId_ = DistributedDB::DATA_SYNC_MESSAGE; - info.messageType_ = DistributedDB::TYPE_REQUEST; - info.sendCode_ = -DistributedDB::E_UNFINISHED; - info.mode_ = DistributedDB::PUSH; - } - - int BuildPushMessage(DataSyncMessageInfo &info, DistributedDB::Message *&message) - { - return DistributedDBToolsUnitTest::BuildMessage(info, message); - } -} - -/** - * @tc.name: SlidingWindow001 - * @tc.desc: Test sliding window receive data when water error happen - * @tc.type: FUNC - * @tc.require: AR000EV1G6 - * @tc.author: zhangqiquan - */ -HWTEST_F(DistributedDBSyncModuleTest, SlidingWindow001, TestSize.Level1) -{ - std::shared_ptr mockSingleVerDataSync = std::make_shared(); - auto *context = new MockSingleVerSyncTaskContext(); - - SlidingWindowReceiver receiver; - receiver.Initialize(context, mockSingleVerDataSync); - uint32_t sequenceId = 1; - uint32_t sessionId = 1; - uint64_t packedId = 1; - DataSyncMessageInfo info{}; - InitMessage(info); - info.sequenceId_ = sequenceId; - info.sessionId_ = sessionId; - info.packetId_ = packedId; - - DistributedDB::Message *message = nullptr; - int status = BuildPushMessage(info, message); - EXPECT_EQ(status, E_OK); - - /** - * @tc.steps: step1. sliding window receive data first and now water error happen - * @tc.expected: step1. receive msg should return -E_NOT_NEED_DELETE_MSG. - */ - EXPECT_CALL(*context, HandleDataRequestRecv(testing::_)).WillRepeatedly(testing::Return(E_OK)); - EXPECT_CALL(*static_cast(mockSingleVerDataSync.get()), - SendDataAck(testing::_, testing::_, testing::_, testing::_)).WillRepeatedly(testing::Return(E_OK)); - context->SetReceiveWaterMarkErr(true); - EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); - - /** - * @tc.steps: step2. sliding window receive data which is has diff sessionId - * @tc.expected: step2. receive msg should return -E_NOT_NEED_DELETE_MSG. - */ - info.sessionId_++; - status = BuildPushMessage(info, message); - EXPECT_EQ(status, E_OK); - EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); - - /** - * @tc.steps: step3. sliding window receive data which is has diff sequenceId - * @tc.expected: step3. receive msg should return -E_NOT_NEED_DELETE_MSG. - */ - info.sequenceId_++; - info.packetId_++; - status = BuildPushMessage(info, message); - EXPECT_EQ(status, E_OK); - EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); - - receiver.Clear(); - // ensure to call DecRefObj in another thread - std::this_thread::sleep_for(std::chrono::seconds(1)); - RefObject::KillAndDecObjRef(context); - context = nullptr; -} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp deleted file mode 100644 index eaa1f7764c710163d2eeaf4967bdfb1a492d3c95..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "virtual_device.h" - -#include -#include - -#include "device_manager.h" -#include "log_print.h" -#include "multi_ver_sync_state_machine.h" -#include "single_ver_subscribe_manager.h" -#include "single_ver_sync_state_machine.h" -#include "sync_types.h" -#include "virtual_communicator.h" -#include "virtual_communicator_aggregator.h" -#include "virtual_multi_ver_sync_db_interface.h" -#include "virtual_single_ver_sync_db_Interface.h" - -namespace DistributedDB { -VirtualDevice::VirtualDevice(const std::string &deviceId) - : communicateHandle_(nullptr), - communicatorAggregator_(nullptr), - storage_(nullptr), - metadata_(nullptr), - deviceId_(deviceId), - remoteDeviceId_("real_device"), - context_(nullptr) -{ -} - -VirtualDevice::~VirtualDevice() -{ - std::mutex cvMutex; - std::condition_variable cv; - bool finished = false; - Offline(); - - if (communicateHandle_ != nullptr) { - communicateHandle_->RegOnMessageCallback(nullptr, nullptr); - communicatorAggregator_->ReleaseCommunicator(communicateHandle_); - communicateHandle_ = nullptr; - } - communicatorAggregator_ = nullptr; - - if (context_ != nullptr) { - IKvDBSyncInterface *storage = storage_; - context_->OnLastRef([storage, &cv, &cvMutex, &finished]() { - if (storage != nullptr) { - delete storage; - } - { - std::lock_guard lock(cvMutex); - finished = true; - } - cv.notify_one(); - }); - RefObject::KillAndDecObjRef(context_); - std::unique_lock lock(cvMutex); - cv.wait(lock, [&finished] {return finished;}); - } else { - delete storage_; - } - context_ = nullptr; - metadata_ = nullptr; - storage_ = nullptr; - subManager_ = nullptr; -} - -int VirtualDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface) -{ - if ((communicatorAggregator == nullptr) || (syncInterface == nullptr)) { - return -E_INVALID_ARGS; - } - - communicatorAggregator_ = communicatorAggregator; - int errCode = E_OK; - communicateHandle_ = communicatorAggregator_->AllocCommunicator(deviceId_, errCode); - if (communicateHandle_ == nullptr) { - return errCode; - } - - storage_ = syncInterface; - metadata_ = std::make_shared(); - if (metadata_->Initialize(storage_) != E_OK) { - LOGE("metadata_ init failed"); - return -E_NOT_SUPPORT; - } - if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_SVD) { - context_ = new (std::nothrow) SingleVerSyncTaskContext; - subManager_ = std::make_shared(); - static_cast(context_)->SetSubscribeManager(subManager_); - } else { - context_ = new (std::nothrow) MultiVerSyncTaskContext; - } - if (context_ == nullptr) { - return -E_OUT_OF_MEMORY; - } - communicateHandle_->RegOnMessageCallback(std::bind(&VirtualDevice::MessageCallback, this, - std::placeholders::_1, std::placeholders::_2), []() {}); - context_->Initialize(remoteDeviceId_, storage_, metadata_, communicateHandle_); - context_->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); - context_->RegOnSyncTask(std::bind(&VirtualDevice::StartResponseTask, this)); - return E_OK; -} - -void VirtualDevice::SetDeviceId(const std::string &deviceId) -{ - deviceId_ = deviceId; -} - -std::string VirtualDevice::GetDeviceId() const -{ - return deviceId_; -} - -int VirtualDevice::GetData(const Key &key, VirtualDataItem &item) -{ - VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); - return syncAble->GetSyncData(key, item); -} - -int VirtualDevice::GetData(const Key &key, Value &value) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->GetData(key, value); -} - -int VirtualDevice::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) -{ - VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); - LOGI("dev %s put data time %llu", deviceId_.c_str(), time); - return syncAble->PutData(key, value, time, flag); -} - -int VirtualDevice::PutData(const Key &key, const Value &value) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->PutData(key, value); -} - -int VirtualDevice::DeleteData(const Key &key) -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->DeleteData(key); -} - -int VirtualDevice::StartTransaction() -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->StartTransaction(); -} - -int VirtualDevice::Commit() -{ - VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); - return syncInterface->Commit(); -} - -int VirtualDevice::MessageCallback(const std::string &deviceId, Message *inMsg) -{ - if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { - if (onRemoteDataChanged_) { - onRemoteDataChanged_(deviceId); - delete inMsg; - inMsg = nullptr; - return E_OK; - } - delete inMsg; - inMsg = nullptr; - return -E_INVALID_ARGS; - } - - LOGD("[VirtualDevice] onMessag, src %s", deviceId.c_str()); - RefObject::IncObjRef(context_); - RefObject::IncObjRef(communicateHandle_); - SyncTaskContext *context = context_; - ICommunicator *communicateHandle = communicateHandle_; - std::thread thread([context, communicateHandle, inMsg]() { - int errCode = context->ReceiveMessageCallback(inMsg); - if (errCode != -E_NOT_NEED_DELETE_MSG) { - delete inMsg; - } - RefObject::DecObjRef(context); - RefObject::DecObjRef(communicateHandle); - }); - thread.detach(); - return E_OK; -} - -void VirtualDevice::OnRemoteDataChanged(const std::function &callback) -{ - onRemoteDataChanged_ = callback; -} - -void VirtualDevice::Online() -{ - static_cast(communicateHandle_)->Enable(); - communicatorAggregator_->OnlineDevice(deviceId_); -} - -void VirtualDevice::Offline() -{ - static_cast(communicateHandle_)->Disable(); - communicatorAggregator_->OfflineDevice(deviceId_); -} - -int VirtualDevice::StartResponseTask() -{ - LOGD("[VirtualDevice] StartResponseTask"); - RefObject::AutoLock lockGuard(context_); - int status = context_->GetTaskExecStatus(); - if ((status == SyncTaskContext::RUNNING) || context_->IsKilled()) { - LOGD("[VirtualDevice] StartResponseTask status:%d", status); - return -E_NOT_SUPPORT; - } - if (context_->IsTargetQueueEmpty()) { - LOGD("[VirtualDevice] StartResponseTask IsTargetQueueEmpty is empty"); - return E_OK; - } - context_->SetTaskExecStatus(ISyncTaskContext::RUNNING); - context_->MoveToNextTarget(); - LOGI("[VirtualDevice] machine StartSync"); - context_->UnlockObj(); - int errCode = context_->StartStateMachine(); - context_->LockObj(); - if (errCode != E_OK) { - LOGE("[VirtualDevice] machine StartSync failed"); - context_->SetOperationStatus(SyncOperation::OP_FAILED); - } - return errCode; -} - -TimeOffset VirtualDevice::GetLocalTimeOffset() const -{ - return metadata_->GetLocalTimeOffset(); -} - -void VirtualDevice::SetSaveDataDelayTime(uint64_t milliDelayTime) -{ - VirtualSingleVerSyncDBInterface *syncInterface = static_cast(storage_); - syncInterface->SetSaveDataDelayTime(milliDelayTime); -} - -int VirtualDevice::Sync(SyncMode mode, bool wait) -{ - auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} - -int VirtualDevice::Subscribe(QuerySyncObject query, bool wait, int id) -{ - auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, SUBSCRIBE_QUERY, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - operation->SetQuery(query); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} - -int VirtualDevice::UnSubscribe(QuerySyncObject query, bool wait, int id) -{ - auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, UNSUBSCRIBE_QUERY, nullptr, wait); - if (operation == nullptr) { - return -E_OUT_OF_MEMORY; - } - operation->Initialize(); - operation->SetOnSyncFinished([operation](int id) { - operation->NotifyIfNeed(); - }); - operation->SetQuery(query); - context_->AddSyncOperation(operation); - operation->WaitIfNeed(); - RefObject::KillAndDecObjRef(operation); - return E_OK; -} -} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h deleted file mode 100644 index f6d93ff9916609c1876b90af6a5d22ab95fe441d..0000000000000000000000000000000000000000 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2021 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef VIRTUAL_DEVICE_H -#define VIRTUAL_DEVICE_H - -#include "icommunicator.h" - -#include "ikvdb_sync_interface.h" -#include "single_ver_sync_task_context.h" -#include "virtual_single_ver_sync_db_Interface.h" - -namespace DistributedDB { -class VirtualCommunicatorAggregator; - -class VirtualDevice { -public: - explicit VirtualDevice(const std::string &deviceId); - ~VirtualDevice(); - - int Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface); - void SetDeviceId(const std::string &deviceId); - std::string GetDeviceId() const; - int GetData(const Key &key, VirtualDataItem &item); - int GetData(const Key &key, Value &value); - int PutData(const Key &key, const Value &value, const TimeStamp &time, int flag); - int PutData(const Key &key, const Value &value); - int DeleteData(const Key &key); - int StartTransaction(); - int Commit(); - int MessageCallback(const std::string &deviceId, Message *inMsg); - void OnRemoteDataChanged(const std::function &callback); - void Online(); - void Offline(); - int StartResponseTask(); - TimeOffset GetLocalTimeOffset() const; - void SetSaveDataDelayTime(uint64_t milliDelayTime); - int Sync(SyncMode mode, bool wait); - int Subscribe(QuerySyncObject query, bool wait, int id); - int UnSubscribe(QuerySyncObject query, bool wait, int id); - -private: - ICommunicator *communicateHandle_; - VirtualCommunicatorAggregator *communicatorAggregator_; - IKvDBSyncInterface *storage_; - std::shared_ptr metadata_; - std::string deviceId_; - std::string remoteDeviceId_; - SyncTaskContext *context_; - std::function onRemoteDataChanged_; - std::shared_ptr subManager_; -}; -} // namespace DistributedDB - -#endif // VIRTUAL_DEVICE_H diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h new file mode 100644 index 0000000000000000000000000000000000000000..c891afa0b8f944d43e64b79428770f53b345a793 --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/include/distributed_rdb_tools.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H +#define DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H +#include +#include + +#include "relational_store_delegate.h" +#include "relational_store_manager.h" +#include "types.h" +#include "distributed_test_sysinfo.h" +#include "distributeddb_data_generator.h" +#include "log_print.h" +#ifdef TESTCASES_USING_GTEST +#define HWTEST_F(test_case_name, test_name, level) TEST_F(test_case_name, test_name) +#endif + +struct RdbParameters { + std::string path; + std::string storeId; + std::string appId; + std::string userId; + RdbParameters(str::string pathStr, std::string storeIdStr, std::string appIdStr, std::string userIdStr) + : pathStr(pathStr), storeId(storeIdStr), appId(appIdStr), userId(userIdStr) + { + } +}; + +const static std::string TAG = "DistributedRdbTools"; + +const static std::string NORMAL_PATH = "/data/test/"; +const static std::string NON_EXISTENT_PATH = "/data/test/nonExistent_rdb/"; +const static std::string UNREADABLE_PATH = "/data/test/unreadable_rdb/"; +const static std::string UNWRITABLE_PATH = "/data/test/unwritable_rdb/"; + +const static std::string NULL_STOREID = ""; +const static std::string ILLEGAL_STOREID = "rdb_$%#@~%"; +const static std::string MODE_STOREID = "rdb_mode"; +const static std::string FULL_STOREID = "rdb_full"; +const static std::string SUPER_STOREID = "rdb_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +const static std::string NON_EXISTENT_TABLE = "non_table"; +const static std::string KEYWORD_START_TABLE = "naturalbase_rdb_a"; + +const static std::string NORMAL_TABLE = "NORMAL_RDB"; +const static std::string CONSTRAINT_TABLE = "CONSTRAINT_RDB"; + +const static RdbParameters g_rdbParameter1(NORMAL_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter2(NORMAL_PATH, DistributedDBDataGenerator::STORE_ID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter3(NORMAL_PATH, ILLEGAL_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter4(NORMAL_PATH, MODE_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter5(NORMAL_PATH, SUPER_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter6(NON_EXISTENT_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter7(UNREADABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter8(UNWRITABLE_PATH, DistributedDBDataGenerator::STORE_ID_1, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +const static RdbParameters g_rdbParameter9(NORMAL_PATH, FULL_STOREID, + DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); + +class DistributedRdbTools final { +public: + DistributedRdbTools() {} + ~DistributedRdbTools() {} + // Relational Database OpenStore And CreateDistributeTable + static DistributedDB::DBStatus GetOpenStoreStatus(const RelatetionalStoreManager *&manager, + RelatetionalStoreDelegate *&delegate, const RdbParameters ¶m); + static DistributedDB::DBStatus GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, + const std::string &tableName); + static bool CloseStore(const DistributedDB::RelatetionalStoreDelegate *&delegate); + + static bool InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m); + static bool InitTableDataAndTrigger(const sqlite3 *&db) ; + static bool AlterTableAttributes(const sqlite3 *&db); + static bool Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name); + static void CloseSqlite3Store(sqlite3 *&db); +} +#endif // DISTRIBUTED_RDB_MODULE_TEST_TOOLS_H \ No newline at end of file diff --git a/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fae08163e170eaa7eca056e45d3285a89e15a562 --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/src/distributed_rdb_tools.cpp @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "distributed_rdb_tools.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef USE_SQLITE_SYMBOLS +#include "sqlite3.h" +#else +#include "sqlite3sym.h" +#endif +#include "distributeddb_data_generator.h" +#include "distributed_test_sysinfo.h" +#include "platform_specific.h" +#include "securec.h" + +using namespace std; +using namespace DistributedDB; +using namespace DistributedDBDataGenerator; + +namespace { + const int MAX_DISTRIBUTED_TABLE_COUNT = 32; + std::string gtableNameList[MAX_DISTRIBUTED_TABLE_COUNT + 1]; + const char *SQL_CREATE_NORMAL_TABLE = "CREATE TABLE IF NOT EXISTS NORMAL_RDB(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "name VARCHAR(100) NOT NULL DEFAULT \"rdb\");"; + + const char *SQL_CREATE_CONSTRAINT_TABLE = "CREATE TABLE IF NOT EXISTS CONSTRAINT_RDB(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "f_id INT," \ + "age INT NOT NULL UNIQUE," \ + "name VARCHAR(100) NOT NULL DEFAULT \"rdb\"," \ + "address CHAR(50) COLLATE NOCASE," \ + "identity INT NOT NULL, CHECK(identity > 10000) PRIMARY KEY(id, identity)," \ + "FOREGIN key(f_id) references NORMAL_RDB(id));"; + + const char *SQL_CREATE_TRIGGER = "CREATE TRIGGER IF NOT EXISTS insertTrigger " \ + "AFTER INSERT ON CONSTRAINT_RDB " \ + "FOR EACH ROW " \ + "BEGIN " \ + "update NORMAL_RDB set name = \"name_001\" " \ + "END"; + + const char *SQL_INSERT_NORMAL_TABLE = "INSERT INTO NORMAL_RDB (id,name)" \ + "VALUES (1, \'rdb_001\'), (2, \'rdb_002\'), (3, \'rdb_003\'), (4, \'rdb_004\'), (5, \'rdb_005\');"; + + + const char *SQL_ADD_FIELD_TABLE1 = "ALTER TABLE RDB_1 ADD COLUMN add_id INI"; + + const char *SQL_ADD_INDEX_TABLE2 = "CREATE INDEX name_index RDB_2 (name)"; + + const char *SQL_DROP_TABLE3 = "DROP TABLE RDB_3"; + + const char *SQL_DROP_CREATE_TABLE3 = "CREATE TABLE IF NOT EXISTS RDB_3(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT);"; + + const char *SQL_DROP_TABLE4 = "DROP TABLE RDB_4"; + + const char *SQL_DROP_CREATE_TABLE4 = "CREATE TABLE IF NOT EXISTS RDB_4(" \ + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, name CHAR(100));"; + + const char *SQL_JOURNAL_MODE = "PRAGMA journal_mode = DELETE;"; + + const char *SQL_SYNCHRONOUS_MODE = "PRAGMA synchronous = FULL;"; + + const char *SQL_INSERT_RDB5_TABLE = "INSERT INTO RDB_5 (id,name,age)" \ + "VALUES (1, \'rdb_005\', \'name_rdb5\');"; +} + +DBStatus DistributedRdbTools::GetOpenStoreStatus(const RelatetionalStoreManager *&manager, + RelatetionalStoreDelegate *&delegate, const RdbParameters ¶m) +{ + if (manager == nullptr) { + MST_LOG("%s GetRdbStore failed! manager nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + if (delegate != nullptr) { + delegate = nullptr; + } + DBStatus status = manager->OpenStore(param.path, param.storeId, delegate); + if (delegate == nullptr) { + MST_LOG("%s GetRdbStore failed! delegate nullptr.", TAG.c_str()); + } + retrun status; +} + +DBStatus DistributedRdbTools::GetCreateDistributedTableStatus(const RelatetionalStoreDelegate *&delegate, + const std::string &tableName) +{ + if (delegate == nullptr) { + MST_LOG("%s CreateDistributedTable failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + DBStatus status = delegate->CreateDistributedTable(tableName); + retrun status; +} + +DBStatus DistributedRdbTools::CloseStore(const RelatetionalStoreDelegate *&delegate) +{ + if (delegate == nullptr) { + MST_LOG("%s CloseStore failed! delegate nullptr.", TAG.c_str()); + return DBStatus::DB_ERROR; + } + DBStatus status = delegate->CloseStore(); + return status; +} + +bool DistributedRdbTools::InitSqlite3Store(sqlite3 *&db, const RdbParameters ¶m) +{ + const std::string dbName = param.path + param.storeId + ".db"; + DBStributedDB::OS::RemoveFile(dbName); + int errCode = sqlite3_open(dbName, &db); + if (errCode1 != SQLITE_OK) { + MST_LOG("sqlite3_open Failed!"); + return false; + } + errCode = sqlite3_exec(db, "PRAGMA journal_mode = WAL;", 0, 0, 0); + if (errCode != SQLITE_OK) { + MST_LOG("PRAGMA journal_mode = WAL Failed!"); + return false; + } + return true; +} + +namespace { +void SqliteExecSql(sqlite3 *db, const char *sql) +{ + char *errMsg = nullptr; + int errCode = sqlite3_exec(db, sql, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec sql Failed(%s)", errMsg.c_str()); + return false; + } + sqlite3_free(errMsg); + errMsg = nullptr; +} +} + +bool DistributedRdbTools::InitTableDataAndTrigger(const sqlite3 *&db) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + SqliteExecSql(db, SQL_CREATE_NORMAL_TABLE); + SqliteExecSql(db, SQL_CREATE_CONSTRAINT_TABLE); + SqliteExecSql(db, SQL_CREATE_TRIGGER); + SqliteExecSql(db, SQL_INSERT_NORMAL_TABLE); + + for (int i = 1; i <= MAX_DISTRIBUTED_TABLE_COUNT + 1; i++) { + std::string str_0 = "RDB_" + i; + std::string str_1 = "CREATE TABLE IF NOT EXISTS " + std::string str_2 = "( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT," \ + "name VARCHAR(100) NOT NULL, age VARCHAR(100) NOT NULL);"; + std::string sql = str_1 + str_0 + str_2; + + SqliteExecSql(db, sql.c_str()); + gtableNameList[i-1] = str_0; + } + return true; +} + +bool DistributedRdbTools::AlterTableAttributes(const sqlite3 *&db) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + char *errMsg = nullptr; + int errCode = sqlite3_exec(db, SQL_ADD_FIELD_TABLE1, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_FIELD_TABLE1 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_ADD_INDEX_TABLE2, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_INDEX_TABLE2 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_TABLE3, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_TABLE3 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_CREATE_TABLE3, nullptr, nullptr, &errMsg); + if (errCode1 != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_ADD_INDEX_TABLE Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_TABLE4, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_TABLE4 Failed(%s)", errMsg.c_str()); + return false; + } + + errCode = sqlite3_exec(db, SQL_DROP_CREATE_TABLE4, nullptr, nullptr, &errMsg); + if (errCode != SQLITE_OK && errMsg != nullptr) { + MST_LOG("sqlite3_exec SQL_DROP_CREATE_TABLE4 Failed(%s)", errMsg.c_str()); + return false; + } + return true; +} + + +bool DistributedRdbTools::Sqlite3ExecOpration(const sqlite3 *&db, cont char *&sql_name) +{ + if (db == nullptr) { + MST_LOG("openStore Failed"); + return false; + } + int errCode = sqlite3_exec(db, sql_name, 0, 0, 0); + if (errCode != SQLITE_OK) { + MST_LOG("%s Failed!", sql_name); + return false; + } + return true; +} + +void DistributedRdbTools::CloseSqlite3Store(sqlite3 *&db) +{ + if (db != nullptr) { + sqlite3_close(db); + db = nullptr; + } +} \ No newline at end of file diff --git a/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..530baf215e5fa37d669efd448898fe5632b139b4 --- /dev/null +++ b/services/distributeddataservice/test/moduletest/common/distributeddb/src/distributed_rdb_exception_test.cpp @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "relational_store_delegate.h" +#include "relational_store_manager.h" +#include "distributed_rdb_tools.h" + +using namespace std; +using namespace testing; +#if defined TESTCASES_USING_GTEST_EXT +using namespace testing::ext; +#endif +using namespace DistributedDB; +using namespace DistributedDBDataGenerator; + +// set sqlite3 rdb_A and create 33 dataTables + +namespace DistributedRelatinalDBExceptionOperation { +sqlite3 *gdb = nullptr; + +RelatetionalStoreManager *g_manager = nullptr; + +RelatetionalStoreDelegate *g_delegate = nullptr; + +RelatetionalStoreDelegate *g_delegate1 = nullptr; + +class DistributedRDBExceptionTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +private: +}; + +void DistributedRDBExceptionTest::SetUpTestCase(void) +{ + bool init1 = DistributedRdbTools::InitSqlite3Store(gdb, g_rdbParameter1); + ASSERT_EQ(init1, true); + bool init2 = DistributedRdbTools::InitTableDataAndTrigger(gdb); + ASSERT_EQ(init2, true); + g_manager = new (std::nothrow) RelatetionalStoreManager(g_rdbParameter1.appId, g_rdbParameter1.userId); + DBStatus status1 = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate, g_rdbParameter1); + ASSERT_EQ(status1, DBStatus::OK); +} + +void DistributedRDBExceptionTest::TearDownTestCase(void) +{ + CloseSqlite3Store(gdb); +} + +void DistributedRDBExceptionTest::SetUp(void) +{ + UnitTest *test = UnitTest::GetInstance(); + ASSERT_NE(test, nullptr); + const TestInfo *testinfo = test->current_test_info(); + ASSERT_NE(testinfo, nullptr); + string testCaseName = string(testinfo->name()); + MST_LOG("[SetUp] test case %s is start to run", testCaseName.c_str()); +} + +void DistributedRDBExceptionTest::TearDown(void) +{ + MST_LOG("TearDownTestCase after case."); +} + +/** + * @tc.name: dbPathException001 + * @tc.desc: db path does not exist, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, dbPathException001, TestSize.Level4) +{ + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter6); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: dbPathException002 + * @tc.desc: db path unreadable, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, dbPathException002, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter7); + ASSERT_EQ(init1, true); + bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IXUSR); + ASSERT_EQ(bpermission, true); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter7); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); + bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter7.path, S_IWUSR | S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + CloseSqlite3Store(db); +} + +/** + * @tc.name: dbPathException003 + * @tc.desc: db path unwritdable, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, dbPathException003, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter8); + ASSERT_EQ(init1, true); + bool bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter8); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); + bpermission = DBStributedDB::OS::SetFilePermissions(g_rdbParameter8.path, S_IWUSR | S_IREAD | S_IXUSR); + ASSERT_EQ(bpermission, true); + CloseSqlite3Store(db); +} + +/** + * @tc.name: storeIdException001 + * @tc.desc: storeId is empty, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, storeIdException001, TestSize.Level4) +{ + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter2); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: storeIdException002 + * @tc.desc: storeId value larger 128, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, storeIdException002, TestSize.Level4) +{ + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter5); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: storeIdException003 + * @tc.desc: storeId is illegal, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, storeIdException003, TestSize.Level4) +{ + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter3); + ASSERT_EQ(status, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: dbModeException001 + * @tc.desc: db mode isn't wal or SYNCHRONOUS full, openStore failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, dbModeException001, TestSize.Level4) +{ + sqlite3 *db = nullptr; + bool init1 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter4); + ASSERT_EQ(init1, true); + bool res1 = DistributedRdbTools::Sqlite3ExecOpration(db, SQL_JOURNAL_MODE); + ASSERT_EQ(res1, true); + DBStatus status1 = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + ASSERT_EQ(status1, DBStatus::NOT_SUPPORT); + CloseSqlite3Store(db); + + bool init2 = DistributedRdbTools::InitSqlite3Store(db, g_rdbParameter9); + ASSERT_EQ(init2, true); + bool res2 = DistributedRdbTools::Sqlite3ExecOpration(db, SQL_SYNCHRONOUS_MODE); + ASSERT_EQ(res2, true); + DBStatus status2 = DistributedRdbTools::GetOpenStoreStatus(g_manager, g_delegate1, g_rdbParameter4); + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); + CloseSqlite3Store(db); +} + +/** + * @tc.name: tableException001 + * @tc.desc: db hasn't non_table, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, tableException001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, NON_EXISTENT_TABLE); + ASSERT_EQ(status2, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: tableException002 + * @tc.desc: db create natrualbase_rdb_A relatinalTable, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, tableException002, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, KEYWORD_START_TABLE); + ASSERT_EQ(status2, DBStatus::INVALID_ARGS); +} + +/** + * @tc.name: tableException003 + * @tc.desc: db NORMAL_RDB table has data, createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, tableException003, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, NORMAL_TABLE); + ASSERT_EQ(status2, DBStatus::NOT_SUPPORT); +} + +/** + * @tc.name: tableMulCreate001 + * @tc.desc: db create RDB_1 RelatinalTable, multiple createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, tableMulCreate001, TestSize.Level4) +{ + for (int i = 1; i <= 5; i++) { + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[0]); + ASSERT_EQ(status2, DBStatus::OK); + } +} + +/** + * @tc.name: mulTableCreate001 + * @tc.desc: dbA createDistributedTable 32 tables, successful + * then dbA createDistributedTable the 33'th table failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, mulTableCreate001, TestSize.Level4) +{ + for (auto tableName: gtableNameList) { + DBStatus status = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, tableName); + if (tableName == gtableNameList[32]) { + ASSERT_EQ(status, DBStatus::NOT_SUPPORT); + } else { + ASSERT_EQ(status, DBStatus::OK); + } + } +} + +/** + * @tc.name: mulTableCreate002 + * @tc.desc: dbA createDistributedTable RDB_1...20 tables, And + * dbB createDistributedTable RDB_21...32 tables,successful + * then dbA createDistributedTable the 33'th table failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, mulTableCreate002, TestSize.Level4) +{ + RelatetionalStoreDelegate *delegateA = nullptr; + RelatetionalStoreDelegate *delegateB = nullptr; + DBStatus status = DistributedRdbTools::GetOpenStoreStatus(g_manager, delegateA, g_rdbParameter1); + ASSERT_EQ(status, DBStatus::OK); + status = DistributedRdbTools::GetOpenStoreStatus(g_manager, delegateB, g_rdbParameter1); + ASSERT_EQ(status, DBStatus::OK); + for (int index = 0; index < gtableNameList.size(); index++) { + if (index < 20) { + status = DistributedRdbTools::GetCreateDistributedTableStatus(delegateA, gtableNameList[index]); + ASSERT_EQ(status, DBStatus::OK); + } else if (index >= 20 && index < 32) { + status = DistributedRdbTools::GetCreateDistributedTableStatus(delegateB, gtableNameList[index]); + ASSERT_EQ(status, DBStatus::OK); + } else { + ASSERT_EQ(status, DBStatus::NOT_SUPPORT); + } + } +} + +/** + * @tc.name: relatinalTable001 + * @tc.desc: db has RDB_1 relatinalTable, then alter RDB_1 and createDistributedTable failed. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, relatinalTable001, TestSize.Level4) +{ + bool result = DistributedRdbTools::AlterTableAttributes(gdb); + ASSERT_EQ(result, true); + for (int i = 0; i < 4; i++) { + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[i]); + ASSERT_EQ(status2, DBStatus::OK); + } +} + +/** + * @tc.name: constraintTable001 + * @tc.desc: db has constraint table, then createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, constraintTable001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, CONSTRAINT_TABLE); + ASSERT_EQ(status2, DBStatus::OK); +} + +/** + * @tc.name: constraintTable001 + * @tc.desc: db has relatinalTable RDB_5, then add data in local RDB_5 and createDistributedTable successful. + * @tc.type: FUNC + * @tc.require: SR000DORPP + * @tc.author: xuhongkang + */ +HWTEST_F(DistributedRDBExceptionTest, constraintTable001, TestSize.Level4) +{ + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + ASSERT_EQ(status2, DBStatus::OK); + bool res1 = DistributedRdbTools::Sqlite3ExecOpration(db, SQL_INSERT_RDB5_TABLE); + ASSERT_EQ(res1, true); + DBStatus status2 = DistributedRdbTools::GetCreateDistributedTableStatus(g_delegate, gtableNameList[4]); + ASSERT_EQ(status2, DBStatus::OK); +} +}